{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm as lgbm\n",
    "from lightgbm.sklearn import LGBMClassifier\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>feat_1_tfidf</th>\n",
       "      <th>feat_2_tfidf</th>\n",
       "      <th>feat_3_tfidf</th>\n",
       "      <th>feat_4_tfidf</th>\n",
       "      <th>feat_5_tfidf</th>\n",
       "      <th>feat_6_tfidf</th>\n",
       "      <th>feat_7_tfidf</th>\n",
       "      <th>feat_8_tfidf</th>\n",
       "      <th>feat_9_tfidf</th>\n",
       "      <th>...</th>\n",
       "      <th>feat_85_tfidf</th>\n",
       "      <th>feat_86_tfidf</th>\n",
       "      <th>feat_87_tfidf</th>\n",
       "      <th>feat_88_tfidf</th>\n",
       "      <th>feat_89_tfidf</th>\n",
       "      <th>feat_90_tfidf</th>\n",
       "      <th>feat_91_tfidf</th>\n",
       "      <th>feat_92_tfidf</th>\n",
       "      <th>feat_93_tfidf</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.081393</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.075886</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.231403</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.199730</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.011987</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011668</td>\n",
       "      <td>0.105971</td>\n",
       "      <td>0.021681</td>\n",
       "      <td>0.080435</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.008244</td>\n",
       "      <td>0.022456</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.124622</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.145988</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Class_1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 95 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  feat_1_tfidf  feat_2_tfidf  feat_3_tfidf  feat_4_tfidf  feat_5_tfidf  \\\n",
       "0   1      0.081393           0.0           0.0      0.000000      0.000000   \n",
       "1   2      0.000000           0.0           0.0      0.000000      0.000000   \n",
       "2   3      0.000000           0.0           0.0      0.000000      0.000000   \n",
       "3   4      0.011987           0.0           0.0      0.011668      0.105971   \n",
       "4   5      0.000000           0.0           0.0      0.000000      0.000000   \n",
       "\n",
       "   feat_6_tfidf  feat_7_tfidf  feat_8_tfidf  feat_9_tfidf   ...     \\\n",
       "0      0.000000      0.000000      0.000000           0.0   ...      \n",
       "1      0.000000      0.000000      0.231403           0.0   ...      \n",
       "2      0.000000      0.000000      0.199730           0.0   ...      \n",
       "3      0.021681      0.080435      0.000000           0.0   ...      \n",
       "4      0.000000      0.000000      0.000000           0.0   ...      \n",
       "\n",
       "   feat_85_tfidf  feat_86_tfidf  feat_87_tfidf  feat_88_tfidf  feat_89_tfidf  \\\n",
       "0       0.075886       0.000000       0.000000            0.0            0.0   \n",
       "1       0.000000       0.000000       0.000000            0.0            0.0   \n",
       "2       0.000000       0.000000       0.000000            0.0            0.0   \n",
       "3       0.000000       0.008244       0.022456            0.0            0.0   \n",
       "4       0.124622       0.000000       0.000000            0.0            0.0   \n",
       "\n",
       "   feat_90_tfidf  feat_91_tfidf  feat_92_tfidf  feat_93_tfidf   target  \n",
       "0       0.000000            0.0            0.0            0.0  Class_1  \n",
       "1       0.000000            0.0            0.0            0.0  Class_1  \n",
       "2       0.000000            0.0            0.0            0.0  Class_1  \n",
       "3       0.000000            0.0            0.0            0.0  Class_1  \n",
       "4       0.145988            0.0            0.0            0.0  Class_1  \n",
       "\n",
       "[5 rows x 95 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#读取数据\n",
    "train=pd.read_csv(\"Otto_FE_train_tfidf.csv\")\n",
    "train.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train=train['target']\n",
    "#只接受数字标签\n",
    "y_train=y_train.map(lambda s :s[6:])\n",
    "y_train=y_train.map(lambda s :int(s)-1)\n",
    "\n",
    "X_train=train.drop([\"id\",\"target\"],axis=1)\n",
    "#特征名字\n",
    "feat_names=X_train.columns\n",
    "#稀疏矩阵\n",
    "from scipy.sparse import csr_matrix\n",
    "X_train=csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_ROUNDS=10000\n",
    "from sklearn.model_selection import  StratifiedKFold#交叉验证\n",
    "\n",
    "kfold=StratifiedKFold(n_splits=3,shuffle=True,random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#内嵌的交叉验证\n",
    "def get_n_estimators(params,X_train,y_train,early_stopping_rounds=10):\n",
    "    lgbm_params=params.copy()\n",
    "    lgbm_params['num_class']=9\n",
    "    lgbmtrain=lgbm.Dataset(X_train,y_train)\n",
    "    \n",
    "    cv_result=lgbm.cv(lgbm_params,lgbmtrain,num_boost_round=MAX_ROUNDS,nfold=3,metrics='multi_logloss',early_stopping_rounds=early_stopping_rounds,seed=3)\n",
    "    \n",
    "    print('best n_estimators:',len(cv_result['multi_logloss-mean']))\n",
    "    print('best cv score:',cv_result['multi_logloss-mean'][-1])\n",
    "    \n",
    "    return len(cv_result['multi_logloss-mean'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best n_estimators: 157\n",
      "best cv score: 0.4864467773524468\n"
     ]
    }
   ],
   "source": [
    "params={'boosting_type':'gbdt',\n",
    "       'objective':'multiclass',\n",
    "       'n_jobs':-1,\n",
    "       'learning_rate':0.1,\n",
    "       'num_leaves':60,\n",
    "       'max_bin':127,\n",
    "       'subsample':0.7,\n",
    "       'bagging_freq':1,\n",
    "       'colsample_bytree':0.7\n",
    "       }\n",
    "n_estimators_1=get_n_estimators(params,X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n",
      "[CV] num_leaves=40 ...................................................\n",
      "[CV] num_leaves=40 ...................................................\n",
      "[CV] num_leaves=40 ...................................................\n",
      "[CV] num_leaves=50 ...................................................\n",
      "[CV] ........ num_leaves=40, score=-0.49740427866680265, total= 1.5min\n",
      "[CV] num_leaves=50 ...................................................\n",
      "[CV] ........ num_leaves=40, score=-0.49304314001565613, total= 1.5min\n",
      "[CV] num_leaves=50 ...................................................\n",
      "[CV] ........ num_leaves=40, score=-0.48399104552609434, total= 1.8min\n",
      "[CV] num_leaves=60 ...................................................\n",
      "[CV] ........ num_leaves=50, score=-0.47759792961911635, total= 1.9min\n",
      "[CV] num_leaves=60 ...................................................\n",
      "[CV] ......... num_leaves=50, score=-0.4946085817831242, total= 1.8min\n",
      "[CV] num_leaves=60 ...................................................\n",
      "[CV] ......... num_leaves=50, score=-0.4868375702083935, total= 2.0min\n",
      "[CV] num_leaves=70 ...................................................\n",
      "[CV] ........ num_leaves=60, score=-0.47657490318900436, total= 2.2min\n",
      "[CV] num_leaves=70 ...................................................\n",
      "[CV] ........ num_leaves=60, score=-0.48598949594184926, total= 2.3min\n",
      "[CV] num_leaves=70 ...................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   8 out of  12 | elapsed:  4.4min remaining:  2.2min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ......... num_leaves=60, score=-0.4939817292924865, total= 2.3min\n",
      "[CV] ......... num_leaves=70, score=-0.4789519654201172, total= 2.3min\n",
      "[CV] ........ num_leaves=70, score=-0.48828654099935015, total= 2.0min\n",
      "[CV] ......... num_leaves=70, score=-0.4942939023207231, total= 1.9min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  12 out of  12 | elapsed:  6.3min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=0.7, importance_type='split', learning_rate=0.1,\n",
       "        max_bin=127, max_depth=-1, maxdepth=7, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_split_gain=0.0, n_estimators=157,\n",
       "     ...ilent=True,\n",
       "        solent=False, subsample=0.7, subsample_for_bin=200000,\n",
       "        subsample_freq=0),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'num_leaves': range(40, 80, 10)},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#num_leaves\n",
    "params={'boosting_type':'gbdt',\n",
    "       'objective':'multiclass',\n",
    "       'n_jobs':-1,\n",
    "       'learning_rate':0.1,\n",
    "       'n_estimators':n_estimators_1,\n",
    "       'num_leaves':60,\n",
    "        'maxdepth':7,\n",
    "       'max_bin':127,\n",
    "       'subsample':0.7,\n",
    "       'bagging_freq':1,\n",
    "       'colsample_bytree':0.7\n",
    "       }\n",
    "lg=LGBMClassifier(solent=False,**params)\n",
    "num_leaves_s=range(40,80,10)\n",
    "tuned_parameters=dict(num_leaves=num_leaves_s)\n",
    "\n",
    "grid_search=GridSearchCV(lg,n_jobs=-1,param_grid=tuned_parameters,cv=kfold,scoring=\"neg_log_loss\",verbose=5,refit=False)\n",
    "grid_search.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4855148135235076\n",
      "{'num_leaves': 60}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/liu/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/liu/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xl8VOXZ//HPlYQACWFNANmDgUBARYngArii4AJdtLWtbW1tbfu4tO7aPm5YW7dWu/D0eWwr3bXW6g9wAXEFVJCgIoYtkX1NWMMWsl2/P+aAkQYyECZnJvm+X695wZy5z8x1e2K+nPs+c25zd0RERI5WUtgFiIhIYlOQiIhIgyhIRESkQRQkIiLSIAoSERFpEAWJiIg0iIJEREQaREEiIiINoiAREZEGSQm7gMaQmZnpffr0CbsMEZGEMn/+/M3unlVfu2YRJH369KGgoCDsMkREEoqZrYqmnYa2RESkQRQkIiLSIAoSERFpEAWJiIg0iIJEREQaREEiIiINoiAREZEGUZAcxhtLSnineHPYZYiIxDUFySFUVdfw0xcXccdzCymvrA67HBGRuKUgOYSU5CTuGzeY1Vv38MTM5WGXIyIStxQkhzGiXyYXndCViW8Us2brnrDLERGJSwqSevzk4jySzPjpi4vCLkVEJC4pSOrRvX1rrjs3h+mFm3hrWWnY5YiIxB0FSRS+MzKb7Mx07p1SyL4qTbyLiNSmIIlCy5Rk7rk0jxWbd/PH2SvCLkdEJK4oSKJ0dm5nLsjrwm9eK2b99r1hlyMiEjcUJEfgrkvyqHHngZcWh12KiEjcUJAcgZ4d0/ivs3N48aMNvK1vvIuIAAqSI/a9s/rSq2Ma90wppKKqJuxyRERCpyA5Qq1aJHP3JXkUl+ziT+9o4l1EJKZBYmZjzGypmRWb2R2HaXeZmbmZ5QfPU81skpktNLMFZnZ2rbYPmNkaM9sVy9oP5/y8Lpw7oDO/erWITWXlYZUhIhIXYhYkZpYMTATGAnnAV8wsr452GcANwNxam78L4O4nAKOBX5jZ/lqnAsNiVXe07rk0j8oa52eaeBeRZi6WZyTDgGJ3X+7uFcDTwPg62t0PPAzU/qd9HvAagLuXANuB/OD5HHffEMO6o9K7UzrfH9WXyR+uZ87yLWGXIyISmlgGSXdgTa3na4NtB5jZyUBPd3/hoH0XAOPNLMXMsoGhQM8Y1npUfnB2Dt3bt+aeyYVUVWviXUSap1gGidWxzQ+8GBmqegy4uY52TxIJngLgceAdoOqIPtzsGjMrMLOC0tLY3COrdWoyd12Sx9JNO/nLu6ti8hkiIvEulkGyls+eRfQA1td6ngEMBt40s5XAacAUM8t39yp3v9Hdh7j7eKA9UHQkH+7uT7h7vrvnZ2VlNagjh3PhoC6M6p/FYzOWUbJTE+8i0vzEMkjmAf3MLNvMUoErgCn7X3T3He6e6e593L0PMAcY5+4FZpZmZukAZjYaqHL3uLyPu5lx76V5lFdV8+DLS8IuR0Sk0cUsSNy9CrgOmA4sBp5x90Izm2Bm4+rZvTPwvpktBm4Hvr7/BTN72MzWAmlmttbM7o1ND6LXN6sN3xnZl+feX0fByq1hlyMi0qjM3etvleDy8/O9oKAgpp+xp6KK837xFh3SUpl6/QiSk+qaIhIRSRxmNt/d8+trp2+2HyNpqSn898V5LNpQxt/nauJdRJoPBckxdNEJXTnj+E48On0pW3btC7scEZFGoSA5hsyMCeMHsaeimoenLQ27HBGRRqEgOcZyOmfw7RHZ/LNgDR+s3hZ2OSIiMacgiYEbzutH54yW3D25kOqapn8xg4g0bwqSGGjTMoWfXDyQhet28M95a+rfQUQkgSlIYmTcSd0Ynt2Rh6cvYdvuirDLERGJGQVJjJgZ940fxM7yKh55RRPvItJ0KUhiaEDXtnzj9N489d5qFq7dEXY5IiIxoSCJsRtH96dTekvumvwxNZp4F5EmSEESY21bteDOsQP4cM12np2/NuxyRESOOQVJI/j8yd0Z2rsDD01bwo49lWGXIyJyTClIGkFSUuQb79v2VPDLGZp4F5GmRUHSSAZ1a8eVp/Xmr3NWUbheE+8i0nQoSBrRzaNzaZ+Wyj2TC2kOt+8XkeZBQdKI2qW14PYxuRSs2sbzH6wLuxwRkWNCQdLILh/akyE92/Ozl5ZQVq6JdxFJfAqSRrZ/4n3L7n08PqMo7HJERBpMQRKCE3u054pTe/Hnd1eydOPOsMsREWkQBUlIbrswl4xWKdw9+WNNvItIQlOQhKRDeiq3XpjL3BVbmbJgfdjliIgcNQVJiK44tRcndG/Hz15azK59VWGXIyJyVBQkIUpOitxqflPZPn7zmibeRSQxKUhCdkqvDnwpvwd/nL2C4hJNvItI4lGQxIHbxwwgLTWZe6boG+8iknhiGiRmNsbMlppZsZndcZh2l5mZm1l+8DzVzCaZ2UIzW2BmZ9dqOzTYXmxmvzYzi2UfGkOnNi25+YJc3i7ewssfbwy7HBGRIxKzIDGzZGAiMBbIA75iZnl1tMsAbgDm1tr8XQB3PwEYDfzCzPbX+jvgGqBf8BgTqz40pq8N78XA49ry0xcWsadCE+8ikjhieUYyDCh29+XuXgE8DYyvo939wMNAea1tecBrAO5eAmwH8s3sOKCtu7/rkTGgvwCfi2EfGk1KchL3jx/E+h3l/Pb14rDLERGJWiyDpDuwptbztcG2A8zsZKCnu79w0L4LgPFmlmJm2cBQoGewf+1lBv/jPWu99zVmVmBmBaWlpQ3rSSPJ79ORL5zSnd/PWs7y0l1hlyMiEpVYBkldcxcHZpKDoarHgJvraPckkZAoAB4H3gGq6nvPz2x0f8Ld8909Pysr6whLD88dYwfQKiWZe6cu0sS7iCSEWAbJWiJnEfv1AGp/hTsDGAy8aWYrgdOAKWaW7+5V7n6juw9x9/FAe6AoeM8eh3nPhNc5oxU/Gt2fmctKeWXRprDLERGpVyyDZB7Qz8yyzSwVuAKYsv9Fd9/h7pnu3sfd+wBzgHHuXmBmaWaWDmBmo4Eqd1/k7huAnWZ2WnC11jeAyTHsQyi+eXpvcrtkMGHqIvZWVIddjojIYcUsSNy9CrgOmA4sBp5x90Izm2Bm4+rZvTPwvpktBm4Hvl7rtR8AfwCKgU+Al4958SFLSU7ivvGDWLd9L79765OwyxEROSxrDuPw+fn5XlBQEHYZR+yGpz5gWuFGZtw4it6d0sMuR0SaGTOb7+759bXTN9vj2E8uHkiLJGPC1EVhlyIickgKkjjWpW0rfnh+P15bUsJrizXxLiLxSUES5646I5vjs9K5b+oiyis18S4i8UdBEudSU5KYMH4wq7fu4YmZy8MuR0TkPyhIEsCZOZlcfMJxTHyjmDVb94RdjojIZyhIEsRPLh5Ikhk/fVET7yISXxQkCaJb+9Zcd24O0ws38dayxLh3mIg0DwqSBPKdkdlkZ6Zz75RC9lVp4l1E4oOCJIG0TEnmnkvzWLF5N3+cvSLsckREAAVJwjk7tzMX5HXhN68Vs3773rDLERFRkCSiuy7Jo8adB15cHHYpIiIKkkTUs2Ma156Tw4sLNzC7aHPY5YhIM6cgSVDXjOpLr45p3DPlYyqqasIuR0SaMQVJgmrVIjLx/knpbv70jibeRSQ8CpIEdt7ALpw3oDO/erWITWXlYZcjIs2UgiTB3X1pHpU1zs9e0sS7iIRDQZLgendK5/uj+jL5w/XMWb4l7HJEpBlSkDQBPzg7h+7tW3PP5EIqqzXxLiKNS0HSBLROTebuS/NYumknf3l3VdjliEgzoyBpIi7I68Ko/lk8PmMZJTs18S4ijUdB0kSYGfdemkd5VTUPvrwk7HJEpBlRkDQhfbPa8N2RfXnu/XUUrNwadjki0kwoSJqY687N4bh2rbhrciHVNR52OSLSDChImpi01BT+++I8Fm8o4+9zNfEuIrFXb5CY2fFm1jL4+9lmdoOZtY/mzc1sjJktNbNiM7vjMO0uMzM3s/zgeQsz+7OZLTSzxWZ2Z622PzSzj82s0Mx+FE0dzc1FJ3TlzJxOPDp9KVt27Qu7HBFp4qI5I/k3UG1mOcAfgWzgH/XtZGbJwERgLJAHfMXM8upolwHcAMyttflyoKW7nwAMBb5nZn3MbDDwXWAYcBJwiZn1i6IPzYqZcd+4QeypqOahaZp4F5HYiiZIaty9Cvg88Li73wgcF8V+w4Bid1/u7hXA08D4OtrdDzwM1L5m1YF0M0sBWgMVQBkwEJjj7nuCmt4K6pKD5HTO4NsjsnmmYC0frN4Wdjki0oRFEySVZvYV4JvAC8G2FlHs1x1YU+v52mDbAWZ2MtDT3V/gs54FdgMbgNXAo+6+FfgYGGVmncwsDbgI6BlFLc3SDef1o0vbltytiXcRiaFoguRbwOnAA+6+wsyygb9FsZ/Vse3AbzMzSwIeA26uo90woBroRmQo7WYz6+vui4GHgBnANGABUFXnh5tdY2YFZlZQWloaRblNT5uWKfz4ooEsXLeDp+etDrscEWmi6g0Sd1/k7je4+1Nm1gHIcPcHo3jvtXz2bKEHsL7W8wxgMPCmma0ETgOmBBPuXwWmuXulu5cAbwP5QT1/dPdT3H0UsBUoOkTdT7h7vrvnZ2VlRVFu0zTupG4Mz+7II9OXsm13RdjliEgTFM1VW2+aWVsz60jkDGCSmf0yiveeB/Qzs2wzSwWuAKbsf9Hdd7h7prv3cfc+wBxgnLsXEBnOOtci0omEzJKgns7Bn72ALwBPHUF/mx0zY8L4wewsr+KRV5aGXY6INEHRDG21c/cyIr+0J7n7UOD8+nYKJsOvA6YDi4Fn3L3QzCaY2bh6dp8ItCEyJzIv+NyPgtf+bWaLgKnAte6umeR65HbN4Jun9+Gp91bz0drtYZcjIk2MuR9+EtbMFgIXAH8GfuLu88zsI3c/sTEKPBby8/O9oKAg7DJCVVZeybmPvkWPDq157gdnkJRU1xSWiMinzGy+u+fX1y6aM5IJRM4qPglCpC+HmJeQ+NW2VQvuHDuAD9ds59n5a8MuR0SakGgm2//l7ie6+w+C58vd/YuxL02OtS+c0p383h14cNoSduypDLscEWkiopls72Fmz5tZiZltMrN/m1mPxihOji0z477xg9i+p4JfzNDEu4gcG9EMbU0icrVVNyJfKJwabJMENKhbO648rTd/m7OKwvU7wi5HRJqAaIIky90nuXtV8PgT0Hy/mNEE3Dw6lw5pqdwzuZD6LrYQEalPNEGy2cyuNLPk4HElsCXWhUnstEtrwe1jBlCwahvPvb8u7HJEJMFFEyTfBr4EbCRy76vLiNw2RRLYZUN7MKRne37+8hLKyjXxLiJHL5qrtla7+zh3z3L3zu7+OSJfTpQElpRkTBg/iC279/H4DF3NLSJH72hXSLzpmFYhoTixR3u+MqwXf353JUs2loVdjogkqKMNEn0tuom49YJcMlqlcLcm3kXkKB1tkOg3ThPRIT2VWy/M5b0VW5myYH39O4iIHOSQQWJmO82srI7HTiLfKZEm4opTe3FC93b87KXF7NpX5/IuIiKHdMggcfcMd29bxyPD3VMas0iJreRg4n1T2T5+/Zom3kXkyBzt0JY0MSf36sCX83vy5OwVFJfsDLscEUkgChI54LYxuaSlJnPPFE28i0j0FCRyQKc2LbnlwlzeLt7CSws3hl2OiCQIBYl8xteG9ybvuLb89MVF7NbEu4hEIZrbyNd19daa4NbyfRujSGk8+yfeN+woZ+IbxWGXIyIJIJozkl8CtxK5hXwP4Bbg98DTwJOxK03Ckt+nI184pTu/n7Wc5aW7wi5HROJcNEEyxt3/z913unuZuz8BXOTu/wQ6xLg+CcmdYwfSKiWZe6cu0sS7iBxWNEFSY2ZfMrOk4PGlWq/pN0wTlZXRkhtH92fmslJeWbQp7HJEJI5FEyRfA74OlASPrwNXmllr4LoY1iYh+8bpvcntksGEqYvYW1EddjkiEqeiuY38cne/1N0zg8el7l7s7nvdfXZjFCnhSElOYsL4QazbvpffvamJdxGpWzRXbfUIrtAqMbNNZvZvM+vRGMVJ+Ib37cT4Id3435nLWbVld9jliEgcimZoaxIwhciNGrsDU4Nt9TKzMWa21MyKzeyOw7S7zMzczPKD5y3M7M9mttDMFpvZnbXa3mhmhWb2sZk9ZWatoqlFjt6PLxpIiyRjwtRFYZciInEomiDJcvdJ7l4VPP4EZNW3k5klAxOBsUAe8BUzy6ujXQZwAzC31ubLgZbufgIwFPiemfUxs+5B23x3HwwkA1dE0QdpgC5tW/HD8/vx2pISXlusiXcR+axogmSzmV1pZsnB40pgSxT7DQOKgzmWCiLfOxlfR7v7gYeB8lrbHEg3sxSgNVAB7F/CLwVoHbyWBmgRjUbwrTOzyenchvumLqK8UhPvIvKpaILk28CXgI3ABuAy4FtR7NcdWFPr+dpg2wFmdjLQ091fOGjfZ4HdweetBh51963uvg54NNi2Adjh7q9EUYs0UIvkJO4bN4jVW/fwxMzlYZcjInEkmqu2Vrv7OHfPcvfO7v454AtRvHddy/Ee+N6JmSUBjwE319FuGFBNZF4mG7jZzPqaWQciZzXZwWvpwRnSf3642TVmVmBmBaWlpVGUK/U5MyeTi084jolvFLNm656wyxGROHG0N228KYo2a4GetZ734LPDUBnAYOBNM1sJnAZMCSbcvwpMc/dKdy8B3gbygfOBFe5e6u6VwHPAGXV9uLs/4e757p6flVXvlI5E6ScXDyTJjPtf0MS7iEQcbZDUdbZxsHlAPzPLNrNUIpPiU/a/6O47gu+l9HH3PsAcYJy7FxAZujrXItKJhMySYPtpZpZmZgacByw+yj7IUejWvjXXn5fDK4s28ebSkrDLEZE4cLRBUu+tUdy9isg336cT+WX/jLsXmtkEMxtXz+4TgTbAx0QCaZK7f+Tuc4nMn7wPLAzqf+Io+yBH6eoR2WRnpnPf1EXsq9LEu0hzZ4e6IZ+Z7aTuwDCgdSKt256fn+8FBQVhl9GkvLWslG8++R63XpjLtefkhF2OiMSAmc139/z62h3yjMTdM9y9bR2PjEQKEYmNs/pnceGgLvz29WLWbd8bdjkiEiKtkChH7a5L8qhx52cvappKpDlTkMhR69EhjWvPyeHFhRuYXbQ57HJEJCQKEmmQa0b1pXenNO6Z8jEVVTVhlyMiIVCQSIO0apHMPZfm8Unpbv70zoqwyxGREChIpMHOHdCF8wZ05levFrGprLz+HUSkSVGQyDFxz6WDqKxxHtDEu0izoyCRY6JXpzS+f9bxTFmwnnc/iebm0CLSVChI5Jj5r7OPp0eH1tw7pZDKak28izQXChI5Zlq1SOauS/JYumknf3l3VdjliEgjUZDIMXVBXhfO6p/F4zOWUbJTE+8izYGCRI4pM+PecYPYV1XDgy8vCbscEWkEChI55rIz0/nOyGyee38dBSu3hl2OiMSYgkRi4rpzc+jWrhV3TS6kShPvIk2agkRiIi01hf++JI/FG8r4+9zVYZcjIjGkIJGYGTu4KyNyMvnFK0vZvGtf2OWISIwoSCRmIhPveeypqObhaZp4F2mqFCQSUzmdM7h6RDbPFKzl/dXbwi5HRGJAQSIxd/15/ejStiX3TC6kuqbupZ1FJHEpSCTm2rRM4ccXDWThuh08PU8T7yJNjYJEGsW4k7oxPLsjj0xfyrbdFWGXI9IsNNYIgIJEGoWZMWH8YHaWV/Hw9KVhlyPSpO0sr+TR6Uu58PGZlFdWx/zzFCTSaHK7ZnDVGX14et5qPlq7PexyRJqciqoaJr29grMeeZPfvlHMgK4Z7KmIfZCkxPwTRGr54fn9mPzheu6eXMhzPziDpCQLuySRhFdT47ywcAOPTl/K6q17OL1vJ+68aAAn9mjfKJ8f0zMSMxtjZkvNrNjM7jhMu8vMzM0sP3jewsz+bGYLzWyxmd0ZbM81sw9rPcrM7Eex7IMcW21bteDHFw3gwzXb+df8NWGXI5Lw3inezPiJb3PDUx+QlprMpG+dyj++O7zRQgRieEZiZsnARGA0sBaYZ2ZT3H3RQe0ygBuAubU2Xw60dPcTzCwNWGRmT7n7UmBIrfdfBzwfqz5IbHz+5O489d5qHpq2lDGDjqNdWouwSxJJOIs3lPHgy0t4a1kp3dq14tHLT+LzJ3cnOYSz/FiekQwDit19ubtXAE8D4+todz/wMFB78QoH0s0sBWgNVABlB+13HvCJu2sFpQRjZtw3bjDb91TwixmaeBc5Euu27+WmZz7kol/P4oPV27hz7ABev+VsLhvaI5QQgdjOkXQHao9drAWG125gZicDPd39BTO7pdZLzxIJnQ1AGnCjux98P/IrgKeOedXSKPK6teXrp/Xmr3NW8eVTezKoW7uwSxKJazv2VDLxzWL+9M5KAL47si//dfbxtE9LDbcwYhskdUXjgYuazSwJeAy4qo52w4BqoBvQAZhlZq+6+/Jg31RgHHDnIT/c7BrgGoBevXodXQ8kpm66IJcXPtrA3ZML+df3TtfEu0gdyiur+fM7K5n4RjE791XxhZN7cNMF/enevnXYpR0QyyBZC/Ss9bwHsL7W8wxgMPCmmQF0BaaY2Tjgq8A0d68ESszsbSAfWB7sOxZ43903HerD3f0J4AmA/Px83ZcjDrVr3YLbxw7gtmc/4vkP1vHFoT3CLkkkblTXOM9/sI5fvrKU9TvKOat/FneMHcDA49qGXdp/iOUcyTygn5llB2cQVwBT9r/o7jvcPdPd+7h7H2AOMM7dC4DVwLkWkQ6cBtS+fexX0LBWk3DZKT0Y0rM9P395CWXllWGXIxI6d+eNpSVc/OtZ3PKvBXRq05J/fGc4f/72sLgMEYhhkLh7FXAdMB1YDDzj7oVmNiE46ziciUAb4GMigTTJ3T8CCK7iGg08F6vapfEkJRn3jx/Mlt37eGzGsrDLEQnVR2u389Xfz+Vbk+axp6Ka33zlZCZfeyZn5GSGXdphmXvTH/XJz8/3goKCsMuQw/jJ8wt5et4aXrxhBAO6xue/ukRiZdWW3Tz6yjKmLlhPx/RUbjg3h68O701qSrg3HzGz+e6eX187fbNd4sItF+Ty4sLIxPs/rzmNYN5MpEnbsmsfv3m9mL/PXUVyknH9uTlcM6ovGa0S67tVChKJCx3SU7ntwgH8+PmFTFmwnvFDuoddkkjM7Kmo4snZK/jft5azt7KaL+X35Efn96NL21Zhl3ZUFCQSN758ak+enreaB15czHkDu9CmpX48pWmpqq7hX/PX8tiMZZTs3McFeV24bcwAcjq3Cbu0BtHdfyVuJCdFbjVfsnMfv36tKOxyRI4Zd2d64UYufHwmdz63kJ4d03j2+6fzxDfyEz5EQGckEmeG9GzPl/N78uTsFVw+tAf9umSEXZJIg8xftZWfv7SEglXb6JuVzv99fSgX5HVpUvOAOiORuHPbmFzSUpO5Z0ohzeGqQmmaikt28b2/FvDF373Lqq17+NnnT+CVH43iwkFdm1SIgM5IJA51atOSWy/M5a7Jhby0cCMXn3hc2CWJRK2krJzHXi3imYI1tEpJ4ubR/bl6ZDZpqU33123T7ZkktK8O781T763hpy8u4uzcLNI18S5xbte+Kp546xN+P2sFldU1fP203lx3bg6ZbVqGXVrM6f9OiUvJScb9nxvEF3/3Lr99o5jbxwwIuySROlVU1fDUe6v59WtFbNldwSUnHsetF+bSu1N62KU1GgWJxK2hvTvyxVN68IdZy7l8aA/6ZiX+1S3SdLg7Ly7cwCPTl7Jqyx5O69uRJ8cO5KSejbcyYbzQZLvEtTvGDqBVSjL3Tl2kiXeJG+9+soXPTXyb6/7xAa1Skpl01ak89d3TmmWIgM5IJM5lZbTkxtH9mfDCIqYXbmLM4K5hlyTN2JKNZTz08hLeWFrKce1a8chlJ/KFU8JbmTBeKEgk7n3j9N48U7CG+19YxFn9s2idmhx2SdLMrN++l1/OWMa/319LRssU7hg7gKvO6EOrFvpZBAWJJICU5CTuGzeILz8xh9+9WcxNF+SGXZI0Ezv2VPI/bxUz6e2V4PCdEdlce05OXCxvG08UJJIQhvftxPgh3fjfmcv54tAezeqKGGl85ZXV/PXdVfz2jWLKyiv5/JDu3HRBf3p0SAu7tLikIJGE8eOLBvLqok3cNbmQX18xRP8qlGOupsb5fx+u4xevLGPd9r2M6p/FHWMGkNdNa+QcjoJEEkaXtq24+YJcJrywiJPvn8GJ3dsxsl8WI/tlcnKvDqEvAiSJy92ZWbSZB19ewuINZQzu3paHLzuRM+N8ZcJ4oRUSJeG8v3obM5eVMqtoMx+u2U51jZOWmszpfTsxsl8mI/plcXxWepO7n5HExsK1O3hw2mLeLt5Cz46tueWCXC49sRtJzfxKLIh+hUQFiSS0svJK3v1kC7OKSpldtJmVW/YA0K1dK0b0y2RkvyzOzMmkY7qGweSz1mzdwyPTlzJlwXo6pLXg+nP78bXTetEyRVdi7acgqUVB0nys3rKHWcWRUHm7eDNl5VWYweBu7RgZBMspvdvrl0UztnV3Bb95vYi/zYksb3v1iGy+d9bxtE2w5W0bg4KkFgVJ81RVXcPCdTuYVbSZWUWlfLB6O1U1TusWyZzWt+OB+ZWczm00DNYM7K2o5sm3V/C/b37C7oqqYHnb/nRtl5jL2zYGBUktChIB2FleyZzlWw8Mgy3fvBuArm1bBXMrmYzIyaRTM7hba3NSVV3Ds/PX8tiry9hUto/zB3bh9jG5WjQtCgqSWhQkUpc1W/cwu3gzs4s2M7t4Mzv2VgIwqFtbRvbLYlS/TIb26aBhsATl7ry6uISHpi2huGQXp/Rqz50XDeTUPh3DLi1hKEhqUZBIfaprnIXrdjC7qJSZRZt5f9U2qmqcVi2SGJ7d6cD8Sv8uGgZLBO+v3sbPX1rMvJXb6JuZzm1jBnDhoKa1vG1jiIsgMbMxwK+AZOAP7v7gIdpdBvwLONXdC8ysBfAH4BQi33X5i7v/PGjbPnhtMODAt9393cPVoSCRI7VrXxVzl285ML/ySWlFYc/6AAANbElEQVRkGKxzRktG9MtkVHA1WFaGhsHiySelu3hk2lKmFW4ks01LfnR+P758ak9aJOs7Rkcj2iCJ2RcSzSwZmAiMBtYC88xsirsvOqhdBnADMLfW5suBlu5+gpmlAYvM7Cl3X0kkmKa5+2VmlgrongVyzLVpmcJ5A7tw3sAuQOSmfbOLNjOzqJQ3lpTw3PvrABh4XFtGBWcr+X066CZ+ISnZWc6vXi3i6XmR5W1vGt2fq0dka2XNRhLL/8rDgGJ3Xw5gZk8D44FFB7W7H3gYuKXWNgfSzSwFaA1UAGVm1hYYBVwF4O4VwWsiMdWtfWu+dGpPvnRqT2pqnML1ZcwsKmVWUSlPvr2C/5u5nJYpSQzL7sioflmM6JfJgK4ZGkqJsV37qnhi5nL+MGs5FVU1XDm8F9ef169ZLG8bT2IZJN2BNbWerwWG125gZicDPd39BTOrHSTPEgmdDUTOOG50961mNgQoBSaZ2UnAfOCH7r47hv0Q+YykJOOEHu04oUc7rj0nh937qnhvxVZmBleDPfDSYiCylsqInMwDV4R1ztBlpsdKZfWny9tu3lXBxSdElrftk6mbeYYhlkFS1z/FDkzImFkS8BjB2cVBhgHVQDegAzDLzF4lUu8pwPXuPtfMfgXcAdz1Hx9udg1wDUCvXr0a1BGRw0lvmcI5AzpzzoDOAGzYsZdZRZGrwd5aVsrzH0SGwQZ0zTgwaT8su6OGwY6Cu/PSwo08Mn0JK7fsYXh2R/7wzYEMaaYrE8aLmE22m9npwL3ufmHw/E6AWpPm7YBPgF3BLl2BrcA44FvAHHf/a9D2SWAaMDPY3ifYPhK4w90vPlwtmmyXsNTUOIs2lB2YtC9YuY2K6hpSU5IY1qdjcBuXTAZ2bat7O9VjzvIt/PzlJSxYs53cLhncPjaXc3I7a/gwhkK/aiuY31gGnAesA+YBX3X3wkO0fxO4Jbhq63ZgAPBtIkNb84Ar3P0jM5sFfMfdl5rZvUC6u996uFoUJBIv9lZUM3fFlgNnLEs37QQgs00qZ+ZkHvi2fZe2Ggbbb+nGnTw0bQmvLynhuHatuHF0f76o5W0bRehXbbl7lZldB0wncvnvk+5eaGYTgAJ3n3KY3ScCk4CPiQyRTXL3j4LXrgf+HlyxtZzI2YtIQmidmszZuZ05OzcyDLaprDwIlVJmF29m8ofrAejfpc2BUBme3alZLi+8YcdefvlKZHnb9JYp3D5mAN86U8vbxiN9IVEkTtTUOIs3ljG7aDOzijbz3sqtVFTVkJqcRH6fDge+v5J3XNMeBtuxt5LfvfkJk95egTt84/TeXHtODh10B+dGF/rQVjxRkEgiKq+s5r0VkXuDzSrazJKNkWGwjun7h8Eij+PatQ650mNjX9Wny9vu2FvJ54Z056bR/enZUV8VC0voQ1si0jCtWiQzqn8Wo/pnAVBSVn7g3mAzizYzdUFkGCyncxtGBmcrw/t2JC01sf63rqlxJi9Yx6PTI8vbjuyXyR1jBzCoW7uwS5Mo6YxEJAG5O0s37WTWssi37d9bsZV9VTW0SDaG9u5wYH5lULd2cT0pPXNZKQ++vIRFG8oY1K0td44dyIh+Wt42XmhoqxYFiTR15ZXVFKzcdmAYbNGGMgA6pLXgjJxMRgVLEHdvHx/DYB+v28GDLy9hdvFmenRoza0XannbeKQgqUVBIs1N6c59vF28+cD3V0p27gOgb1Z65BYuOZmcdnwn2jTyvajWbN3Do68sZfKHkeVtrzu3H1dqedu4pSCpRUEizZm7U1Syi5nLImcrc1dsobyyhpQk45TeHRiZk8nI/lmc0D12w2Dbdlfwm9eL+ducVSQlwbfPzOb7Z2t523inIKlFQSLyqfLKat5ftY2ZRZuZXVzKx+siw2DtWrfgzJxOjAzOWI7F1VIHL297+dCe3Dhay9smCl21JSJ1atUimTNyMjkjJxMYwJZd+w5cDTaraDMvLdwIQHZm+oF7g53WtyMZR3D2UF3jPDt/DY/NKGJjWTnnD+zM7WMGaHnbJkpnJCJygLtTXLLrwNzKnOVb2VtZTXKScUqv9ozIyWJk/0xO7N6OlDoWi3J3Xl9SwoMvL6GoZBcn92rPnWMHMixby9smIg1t1aIgETk6+6qqeX/VdmYFt3BZuG4H7tC2VQpnHJ/JyP6ZjMzJolenND5YvY2fv7yE91ZsJTszndsuzGXM4K66qWICU5DUoiAROTa27q7g7QPDYKWs31EOQNe2rdhYVk5mm5b88Px+XKHlbZsEzZGIyDHXMT2VS0/qxqUndcPd+aR0N7OLSpm7YisDj2ur5W2bKR1xETkqZkZO5zbkdG7DVWdmh12OhEjnniIi0iAKEhERaRAFiYiINIiCREREGkRBIiIiDaIgERGRBlGQiIhIgyhIRESkQZrFLVLMrBRYdZS7ZwKbj2E5YWoqfWkq/QD1JV41lb40tB+93T2rvkbNIkgawswKornXTCJoKn1pKv0A9SVeNZW+NFY/NLQlIiINoiAREZEGUZDU74mwCziGmkpfmko/QH2JV02lL43SD82RiIhIg+iMREREGkRBchAzSzazD8zsheB5tpnNNbMiM/unmaWGXWO06ujLn8xshZl9GDyGhF1jNMxspZktDGouCLZ1NLMZwXGZYWYdwq4zGofoy71mtq7Wcbko7DrrY2btzexZM1tiZovN7PQEPiZ19SURj0lurXo/NLMyM/tRYxwXBcl/+iGwuNbzh4DH3L0fsA24OpSqjs7BfQG41d2HBI8PwyjqKJ0T1Lz/UsY7gNeC4/Ja8DxRHNwXiPyM7T8uL4VWWfR+BUxz9wHASUR+zhL1mNTVF0iwY+LuS/fXCwwF9gDP0wjHRUFSi5n1AC4G/hA8N+Bc4NmgyZ+Bz4VT3ZE5uC9N0HgixwMS6Lg0BWbWFhgF/BHA3SvcfTsJeEwO05dEdx7wibuvohGOi4Lksx4HbgNqguedgO3uXhU8Xwt0D6Owo3BwX/Z7wMw+MrPHzKxlCHUdDQdeMbP5ZnZNsK2Lu28ACP7sHFp1R6auvgBcFxyXJxNgSKgvUApMCoZO/2Bm6STmMTlUXyCxjsnBrgCeCv4e8+OiIAmY2SVAibvPr725jqZxf5nbIfoCcCcwADgV6Ajc3ti1HaUz3f0UYCxwrZmNCrugBqirL78DjgeGABuAX4RYXzRSgFOA37n7ycBuEmcY62CH6kuiHZMDgnncccC/GuszFSSfOhMYZ2YrgaeJDGk9DrQ3s5SgTQ9gfTjlHZH/6IuZ/c3dN3jEPmASMCzMIqPl7uuDP0uIjPkOAzaZ2XEAwZ8l4VUYvbr64u6b3L3a3WuA3xP/x2UtsNbd5wbPnyXyyzgRj0mdfUnAY1LbWOB9d98UPI/5cVGQBNz9Tnfv4e59iJwWvu7uXwPeAC4Lmn0TmBxSiVE7RF+urPXDZETGST8OscyomFm6mWXs/ztwAZG6pxA5HpAgx+VQfdl/XAKfJ86Pi7tvBNaYWW6w6TxgEQl4TA7Vl0Q7Jgf5Cp8Oa0EjHJeU+ps0e7cDT5vZT4EPCCblEtTfzSyLyJDdh8D3Q64nGl2A5yPZRwrwD3efZmbzgGfM7GpgNXB5iDVG61B9+WtwKbYDK4HvhVdi1K4n8vOUCiwHvkXkH6aJdkyg7r78OgGPCWaWBozms/U+SIyPi77ZLiIiDaKhLRERaRAFiYiINIiCREREGkRBIiIiDaIgERGRBlGQiIhIgyhIREIQ3NL/svpbisQ/BYmIiDSIgkQkYGZ9goWNfm9mhWb2ipm1NrM3zSw/aJMZ3MMMM7vKzP6fmU21yIJh15nZTcFdZOeYWccoP3eomb0V3BF4eq1b2XzXzOaZ2QIz+7eZpZlZO4ssjpUUtEkzszVm1sLMjjezacH7zDKzAUGby83s4+B9ZsbkP540awoSkc/qB0x090HAduCL9bQfDHyVyE39HgD2BHeRfRf4Rn0fZmYtgN8Al7n7UODJ4H0AnnP3U919/2JLV7v7DmABcFbQ5lJgurtXAk8A1wfvcwvwP0Gbu4ELg/cZV19NIkdK99oS+awVtVaOnA/0qaf9G+6+E9hpZjuAqcH2hcCJUXxeLpEwmhHcgyuZyG3LAQYH93hrD7QBpgfb/wl8mcgNRa8A/sfM2gBnAP8K3gdg/3ozbwN/MrNngOeiqEnkiChIRD5rX62/VwOtgSo+PXtvdZj2NbWe1xDd/18GFLr76XW89ifgc+6+wMyuAs4Otk8Bfh4MnQ0FXgfSiSzCNuTgN3H375vZcCIrZn5oZkPcfUsUtYlERUNbIvVbSeQXNny6pMCxshTIMrPTITLUZWaDgtcygA3B8NfX9u/g7ruA94isNf5CsG5GGbDCzC4P3sfM7KTg78e7+1x3vxvYDPQ8xn2QZk5BIlK/R4EfmNk7QOaxfGN3ryASTg+Z2QIit/c/I3j5LmAuMANYctCu/wSuDP7c72vA1cH7FBJZqxvgETNbaGYfAzOJzLGIHDO6jbyIiDSIzkhERKRBNNkuEkNmNhE486DNv3L3SWHUIxILGtoSEZEG0dCWiIg0iIJEREQaREEiIiINoiAREZEGUZCIiEiD/H99QX2W+YtFIgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([-0.49147905, -0.48634748, -0.48551481, -0.48717697])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)\n",
    "\n",
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid_search.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid_search.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid_search.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "n_leafs = len(num_leaves_s)\n",
    "\n",
    "x_axis = num_leaves_s\n",
    "plt.plot(x_axis, -test_means)\n",
    "#plt.errorbar(x_axis, -test_means, yerr=test_stds,label = ' Test')\n",
    "#plt.errorbar(x_axis, -train_means, yerr=train_stds,label = ' Train')\n",
    "plt.xlabel( 'num_leaves' )\n",
    "plt.ylabel( 'Log Loss' )\n",
    "plt.show()\n",
    "test_means"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 'num_leaves': 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 4 candidates, totalling 12 fits\n",
      "[CV] min_child_samples=50 ............................................\n",
      "[CV] min_child_samples=50 ............................................\n",
      "[CV] min_child_samples=50 ............................................\n",
      "[CV] min_child_samples=60 ............................................\n",
      "[CV] .. min_child_samples=50, score=-0.4905558116166891, total= 2.0min\n",
      "[CV] min_child_samples=60 ............................................\n",
      "[CV] .. min_child_samples=60, score=-0.4760216409099392, total= 2.0min\n",
      "[CV] min_child_samples=60 ............................................\n",
      "[CV] .. min_child_samples=50, score=-0.4864231200769947, total= 2.3min\n",
      "[CV] min_child_samples=70 ............................................\n",
      "[CV] .. min_child_samples=50, score=-0.4752052802317861, total= 2.3min\n",
      "[CV] min_child_samples=70 ............................................\n",
      "[CV] .. min_child_samples=70, score=-0.4773609361025619, total= 1.9min\n",
      "[CV] min_child_samples=70 ............................................\n",
      "[CV] . min_child_samples=60, score=-0.48784386089611137, total= 2.2min\n",
      "[CV] min_child_samples=80 ............................................\n",
      "[CV] .. min_child_samples=70, score=-0.4862163029446958, total= 2.1min\n",
      "[CV] min_child_samples=80 ............................................\n",
      "[CV] .. min_child_samples=60, score=-0.4882779483756968, total= 2.5min\n",
      "[CV] min_child_samples=80 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   8 out of  12 | elapsed:  4.8min remaining:  2.4min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] .. min_child_samples=80, score=-0.4766232706062543, total= 2.1min\n",
      "[CV] .. min_child_samples=80, score=-0.4876776661500398, total= 1.9min\n",
      "[CV] . min_child_samples=70, score=-0.49147799104907863, total= 2.3min\n",
      "[CV] . min_child_samples=80, score=-0.48969526315079476, total= 2.0min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  12 out of  12 | elapsed:  6.9min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=0.7, importance_type='split', learning_rate=0.1,\n",
       "        max_bin=127, max_depth=-1, maxdepth=7, min_child_samples=20,\n",
       "        min_child_weight=0.001, min_split_gain=0.0, n_estimators=157,\n",
       "        n_jobs=-1, num_leaves=60, objective='multiclass',\n",
       "        random_state=None, reg_alpha=0.0, reg_lambda=0.0, silent=False,\n",
       "        subsample=0.7, subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'min_child_samples': range(50, 90, 10)},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#min_child_samples\n",
    "params={'boosting_type':'gbdt',\n",
    "       'objective':'multiclass',\n",
    "       'n_jobs':-1,\n",
    "       'learning_rate':0.1,\n",
    "       'n_estimators':n_estimators_1,\n",
    "       'num_leaves':60,\n",
    "        'maxdepth':7,\n",
    "       'max_bin':127,\n",
    "       'subsample':0.7,\n",
    "       'bagging_freq':1,\n",
    "       'colsample_bytree':0.7\n",
    "       }\n",
    "lg=LGBMClassifier(silent=False,**params)\n",
    "min_child_samples_s=range(50,90,10)\n",
    "tuned_parameters=dict(min_child_samples = min_child_samples_s)\n",
    "\n",
    "grid_search=GridSearchCV(lg,n_jobs=-1,param_grid=tuned_parameters,cv=kfold,scoring=\"neg_log_loss\",verbose=5,refit=False)\n",
    "grid_search.fit(X_train,y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4840474205829825\n",
      "{'min_child_samples': 60}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/liu/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/liu/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAD8CAYAAABpcuN4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmcVOWd7/HPrzeWZt+XBpqlQREUpEUUQQRxDzpRIzomGmMwRgSVJKP3zs3MmDt3kowRUIkGNQTHRFxiDCGKgIhsgjQuiCJNAy00IA2yr7397h9VkKJtugvo7lPV9X2/Xry6zjnPOfV7XqeLbz+nqs5j7o6IiMjJJAVdgIiIxDYFhYiIVEpBISIilVJQiIhIpRQUIiJSKQWFiIhUSkEhIiKVUlCIiEilFBQiIlKplKALqA6tWrXyzMzMoMsQEYkrK1eu3OnuratqVyeCIjMzk5ycnKDLEBGJK2b2ZTTtdOlJREQqpaAQEZFKKShERKRSCgoREamUgkJERCqloBARkUopKEREpFIKChGp0vvrv2bp+p1BlyEBiSoozOwqM1trZnlm9nAl7W4yMzez7PByqplNN7NPzWyNmT0S0TY/vP5jM8uJWN/CzOaa2brwz+Zn0kEROTO7DhZx9/QV3Pbscn7+19UcKS4NuiSpZVUGhZklA1OAq4HewK1m1ruCdo2BccDyiNU3A/XcvS8wALjHzDIjtl/m7v3cPTti3cPAO+6eBbwTXhaRgPzuvfUcLi7lpgEZvPD+l1z35GI+27o36LKkFkUzohgI5Ln7BncvAmYA11fQ7hfAr4EjEescSDezFKABUATsq+L5rgemhx9PB26IokYRqQGF+48w/f18bujXkcduPo//+cFA9h0u5oYpS5i6cD1lZR50iVILogmKjsDmiOWC8LrjzKw/0MndZ5Xb9zXgILAN2AQ85u67wtscmGNmK81sTMQ+bd19G0D4Z5toOyMi1eu3766nuNQZNyILgCFZrXn7gaEMP6sN/+/NL7j9+eVs23s44CqlpkUTFFbBuuN/RphZEjARmFBBu4FAKdAB6ApMMLNu4W2D3f18Qpe07jOzoadSuJmNMbMcM8vZsWPHqewqIlHYtvcwf1q+iZvOzyCzVfrx9c3T03jm9gH86sa+fLRpD1dNWsSbn24LsFKpadEERQHQKWI5A9gasdwY6AMsMLN8YBAwM/yG9m3AbHcvdvdCYAmQDeDuW8M/C4G/EAoVgO1m1h4g/LOwoqLcfaq7Z7t7duvWVd4lV0RO0VPz83Cc+0f0+MY2M+OWCzrz5vghZLZsyI//+CE/efUTDhwtCaBSqWnRBMUKIMvMuppZGjAamHlso7vvdfdW7p7p7pnAMmCUu+cQutw03ELSCYXIF2aWHn7zm/D6K4DV4UPOBO4IP74D+OsZ91JETsnmXYd4ecVmRl/QmYzmDU/armurdF6792LGXtaD1z8s4JrJi/hw0+5arFRqQ5VB4e4lwFjgbWAN8Iq7f2Zmj5rZqCp2nwI0IhQCK4Bp7r4KaAssNrNPgA+Av7v77PA+vwRGmtk6YGR4WURq0RPvrCM5yRg7/JujifJSk5P4yZW9ePmeiygtc25+5n0mz1tHSWlZLVQqtcHc4/9TC9nZ2a6Ji0Sqx4YdBxg5cSF3XpzJ/7nuG5+Er9S+I8X8/I3VvPHxVgZ0ac7E7/Sjc8uTj0gkWGa2stzXEyqkb2aLyAkmv7OOtOQk7h3W/ZT3bVI/lUmj+zN5dD9yt+/nmicW8eeVBdSFP0gTmYJCRI7L3b6fmZ9s5c7BmbRqVO+0j3N9v468NX4IvTs0YcKrnzD2pY/Ye6i4GiuV2qSgEJHjJs7NJT0thTFDulXduAoZzRvy0g8H8dMre/H26q+4avJC3l//dTVUKbVNQSEiAHy2dS9vrf6Kuy7pSvP0tGo5ZnKScd9lPXj9xxdTPzWZ255bxn+9tYaiEr3RHU8UFCIChEYTTRuk8oNLulb7sc/NaMbfx13C6As687v3NvDtp5eQV3ig2p9HaoaCQkT4aNNu5q0pZMzQbjRtkFojz9EwLYX/+nZfpn53AFt2H+a6Jxfx4rIv9UZ3HFBQiAiPz82lRXoad16cWePPdcU57Xj7gaFckNmCf31jNT98IYedB47W+PPK6VNQiCS4DzbuYtG6ndx7aXfS66XUynO2aVKf6d8fyM+v683CdTu5atJC3l1b4d16JAYoKEQSmLvzmzlrad24HrcP6lKrz52UZNx1SVdmjh1Mq0b1+P60FfybJkaKSQoKkQS2dP3XLN+4i7GX9aBBWnIgNZzVrglv3DeYuwZ3Zfr7X/KtJxfz+daqpq2R2qSgEElQ7s5jc9bSoWl9Rg/sVPUONah+ajI//1ZvXrhrIHvCEyM9u3CDJkaKEQoKkQS1YO0OPtq0h/tHZFEvJZjRRHlDe4YmRhrWqzX/+eYavvv75Xy190jVO0qNUlCIJCB35zdz19K5RUNuGpARdDknaJGexu++O4BffrsvH365h6smL+QtTYwUKAWFSAJ6+7PtrN6yj3EjskhNjr3/BsyM0QM78/dxl9C5RUPu/eOH/Oy1TzioiZECEXu/ISJSo8rKnIlzc+nWOp0b+nUIupxKdWvdiD/fezH3XdadV1cWcM0Ti/hIEyPVOgWFSIKZ9ek21m7fzwOX9yQlBkcT5aUmJ/HTK8/i5TEXUVLq3PTM+zzxjiZGqk2x/1siItWmpLSMSfNy6dW2Mdf1bR90OadkYNcWvDl+CNed257H5+YyeuoyNu86FHRZCUFBIZJA3vh4Kxt2HOTBkT1JSrKgyzllTRukMnl0fybd0o+1X+3n6smLeP1DTYxU0xQUIgmiuLSMJ95ZR5+OTbjynLZBl3NGbujfkTfHD+Hs9o156JVPGDfjY/Ye1sRINUVBIZIgXltZwKZdh5gwshdm8TeaKK9Ti4bMGHMRP72yF299uo2rJy1k2QZNjFQTFBQiCeBoSSlPvrOO/p2bMaxX66DLqTbHJkb6870XUy81mVufXcavZn+hiZGqmYJCJAHM+GAzW/ceqTOjifLO69SMWfdfwi3ZnXh6wXpufHop63doYqTqoqAQqeMOF5Xy1Lt5XNi1BYN7tAy6nBqTXi+FX954Ls/cPoCC3Ye49olF/HG5JkaqDgoKkTruxWVfsmP/USZcUTdHE+Vd1acds8MTI/3vv6zmhy+s5GtNjHRGFBQiddjBoyU8/d56hmS1YmDXFkGXU2vahidG+j/X9WZh7g6umryIBZoY6bQpKETqsD8szWfXwSImXNEr6FJqXVKS8YNLuvLXsYNp0TCNO6et4N9nfqaJkU6DgkKkjtp3pJipCzcw4qw29OvULOhyAnN2+yb8dexgvj84kz8szWfUU4tZs00TI50KBYVIHfX8oo3sPVzMgyN7Bl1K4OqnJvNv3zqH6XcNZPehYq5/agnPLdLESNFSUIjUQbsPFvH84o1c3acdfTo2DbqcmHFpz9bMHj+EoT1b83//voY7pn3A9n2aGKkqCgqROmjqog0cLCrRaKICLRvV49nvDeD//VNfcvJ3c+Wkhcxe/VXQZcU0BYVIHbNj/1H+sCSfUed1oGfbxkGXE5PMjNsu7MyscZfQqXlDfvTiSh7+8ypNjHQSCgqROuaZ99ZztKSU8SOygi4l5nUPT4z042HdeTlnM9c+sYhPNu8JuqyYE1VQmNlVZrbWzPLM7OFK2t1kZm5m2eHlVDObbmafmtkaM3ukXPtkM/vIzGZFrBthZh+a2cdmttjMepxu50QSzVd7j/Disi+58fwMurVuFHQ5cSEtJYmfXXUWL/1wEEUlZdz49FKemr+OUr3RfVyVQWFmycAU4GqgN3CrmfWuoF1jYBywPGL1zUA9d+8LDADuMbPMiO3jgTXlDvU08M/u3g/4E/Cv0XZGJNFNeTeP0jJnnEYTp2xQt5a89cBQru7bnsfm5DJ66vuaGCksmhHFQCDP3Te4exEwA7i+gna/AH4NRH6EwIF0M0sBGgBFwD4AM8sArgWeK3ccB5qEHzcFtkbXFZHEVrD7EDNWbOKWCzrRqUXDoMuJS00bpPLE6H5MvOU81mzbzzWTF/HGR1uCLitw0QRFR2BzxHJBeN1xZtYf6OTuszjRa8BBYBuwCXjM3XeFt00CfgaUvx/w3cCbZlYAfBf4ZRQ1iiS8J9/Jw8wYO1xXa8+EmfFP/TN4a/wQerVrzAMvf8y4lz5K6ImRogmKiu4idvzinZklAROBCRW0GwiUAh2ArsAEM+tmZtcBhe6+soJ9HgSucfcMYBrweIVFmY0xsxwzy9mxY0cU3RCpu/J3HuS1Dwu4bWBn2jdtEHQ5dUJoYqRBTBjZk79/uo1rJi9ieYJOjBRNUBQAnSKWMzjxclBjoA+wwMzygUHAzPAb2rcBs9292N0LgSVANjAYGBVuPwMYbmYvmllr4Dx3P/Y+x8vAxRUV5e5T3T3b3bNbt647E7GInI4n3llHarLx48u6B11KnZKSnMT9I7J47UcXkZJsjH52Gf/99hcUlybWxEjRBMUKIMvMuppZGjAamHlso7vvdfdW7p7p7pnAMmCUu+cQutw03ELSCYXIF+7+iLtnhNuPBua7++3AbqCpmR37ltBIvvlmt4hEyCvcz18+3sIdF2XSpnH9oMupk/p3bs6b44bwnQGdmPJuaGKkDQk0MVKVQeHuJcBY4G1C/2m/4u6fmdmjZjaqit2nAI2A1YQCZ5q7r6riuX4I/NnMPiH0HsVPo+qJSIKaOG8dDVOTuedSjSZqUnq9FH5107k8c/v5bNp1iGufWMxLH2xKiImRrC50Mjs723NycoIuQ6TWfb51H9c8sYj7h/dIyFuJB+WrvUf4yaufsDhvJyN7t+VXN55Li/S0oMs6ZWa20t2zq2qnb2aLxLGJ83JpXD+Fuy/pFnQpCaVd0/q8cNdA/vXas3lv7Q6unLSQhbl190M1CgqROLWqYA9zP9/OmCHdaNowNehyEk5SknH3kG68cd9gmjVI5Xu//4D/+FvdnBhJQSESp34zJ5fmDVP5/iVdgy4lofXu0IS/3X8Jd16cybQl+Vz/1BK++KpuTYykoBCJQyu/3MV7uTu459LuNKqXEnQ5Ca9+ajL/Puocpn3/Ar4+WMSop5bw+8Ub68zESAoKkTj0mzm5tGpUj+9d1CXoUiTCZb3aMPuBIQzNasWjsz7njmkfUFgHJkZSUIjEmaXrd7J0/df8eFh3GqZpNBFrWjWqx7Pfy+b/3tCHFfm7uHLSQt7+LL4nRlJQiMQRd+fxObm0a1Kf2y7sHHQ5chJmxu2DujDr/iF0bN6Ae/5nJY+8vopDRfE5MZKCQiSOvJe7g5wvdzN2eA/qpyYHXY5UoUebRrx+72B+dGl3ZqzYzHVPLGZVQfxNjKSgEIkT7s7jc3PJaN6A72R3qnoHiQlpKUk8fPVZ/OnuQRwuLuXbv116fN6QeKGgEIkT89YUsqpgL+NGZJGWopduvLmoe0tmjx/KVX3a8d9vr+XWZ5dRsDs+JkbSb5tIHCgrc34zZy1dW6Xz7f4dq95BYlLThqk8eWt/Hv/OeXy+dR9XT17EXz+O/YmRFBQiceCt1V/xxVf7eeDyLFKS9bKNZ2bGt88PTYzUs21jxs/4mAdmfMS+I7E7MZJ+40RiXGmZM3FeLlltGnHduR2CLkeqSacWDXl5zCAevLwnf1u1jasnLeKDjbuq3jEACgqRGDfzky3kFR7gwZE9SU6qaMJJiVcpyUmMvzyLV49NjDT1fR57e23MTYykoBCJYcWlZUyet47e7Ztw1Tntgi5Hasj5nZvz93FDuPH8DJ56N4+bnl7Kxp0Hgy7rOAWFSAx7/cMC8r8+xEMje5Kk0USd1qheCv9983n89p/PJ//rQ1z7xCJeXhEbEyMpKERi1NGSUp54J4/zOjVjxNltgi5Hask1fdsz+4Eh9OvUjH/586f86MWV7D5YFGhNCgqRGPXKis1s2XOYCSN7YqbRRCJp37QBL/7gQv73NWcz/4tCrpy0kEXrgpsYSUEhEoOOFJfy1Lt5XJDZnCFZrYIuRwKQlGT8cGhoYqQmDVL57vMf8ItZnwcyMZKCQiQG/XH5JrbvO8qEK3ppNJHgzunQlL+NvYTvXdSF5xdv5IYpS8jdvr9Wa1BQiMSYQ0UlPL0gj8E9WjKoW8ugy5EY0CAtmUev78O0Oy9g54GjXPfkYqYt2Vhrb3QrKERizPSlX7LzQBEPjewVdCkSYy47qw2zHxjKJT1a8R9/+5w7pq2gcH/NT4ykoBCJIfuPFPO7heu5rFdrBnRpHnQ5EoNaNarH83dk84sb+rBm2z6OFtf8l/M0PZZIDPn94nz2HCrWaEIqZWZ8d1AXbh6QUSvzkmhEIRIj9hwq4rlFG7jynLb0zWgadDkSB2pr8ioFhUiMeHbRBg4UlfDgyJ5BlyJyAgWFSAz4+sBRpi3J59q+7TmrXZOgyxE5gYJCJAb8buEGjhSX8sDlGk1I7FFQiASscN8Rpi/N54b+HenRplHQ5Yh8g4JCJGC/XbCe0jJn/IisoEsRqZCCQiRAW/Yc5k/LN3FzdgZdWqYHXY5IhRQUIgF6an4eAGOHazQhsSuqoDCzq8xsrZnlmdnDlbS7yczczLLDy6lmNt3MPjWzNWb2SLn2yWb2kZnNilhnZvafZpYb3mfc6XZOJJZt+voQr+Zs5taBnejYrEHQ5YicVJXfzDazZGAKMBIoAFaY2Ux3/7xcu8bAOGB5xOqbgXru3tfMGgKfm9lL7p4f3j4eWANEfh7wTqATcJa7l5mZZmyROmnyO+tITjLuu6xH0KWIVCqaEcVAIM/dN7h7ETADuL6Cdr8Afg1E3qHKgXQzSwEaAEXAPgAzywCuBZ4rd5x7gUfdvQzA3Quj745IfFi/4wB/+aiA713UhTZN6gddjkilogmKjsDmiOWC8LrjzKw/0MndZ3Gi14CDwDZgE/CYu+8Kb5sE/Awof0er7sAtZpZjZm+ZWYUXb81sTLhNzo4dwc38JHI6Js1bR/3UZH50afegSxGpUjRBUdGsKcdvgm5mScBEYEIF7QYCpUAHoCswwcy6mdl1QKG7r6xgn3rAEXfPBp4Ffl9RUe4+1d2z3T27devWUXRDJDas/Wo/s1Zt5c6LM2nZqF7Q5YhUKZqgKCD0nsExGcDWiOXGQB9ggZnlA4OAmeE3tG8DZrt7cfgS0hIgGxgMjAq3nwEMN7MXI57vz+HHfwHOPY1+icSsiXNzaZSWwpih3YIuRSQq0QTFCiDLzLqaWRowGph5bKO773X3Vu6e6e6ZwDJglLvnELrcNDz8SaZ0QiHyhbs/4u4Z4fajgfnufnv4kG8Aw8OPLwVyz7ybIrFh9Za9zP7sK34wpCvNGqYFXY5IVKoMCncvAcYCbxP6hNIr7v6ZmT1qZqOq2H0K0AhYTShwprn7qir2+SVwo5l9CvwXcHdVNYrEi8fn5tKsYSp3XdI16FJEohbVxEXu/ibwZrl1Pz9J22ERjw8Q+ohsZcdeACyIWN5D6NNQInXKyi93M/+LQn52VS+a1E8NuhyRqOmb2SK1ZOLcXFqmp3HHRZlBlyJyShQUIrVg2YavWZy3k3uHdSe9nmYglviioBCpYe7O43NyadukHrcP6hJ0OSKnTEEhUsMW5+3kg/xd3HdZj1qb41ikOikoRGqQu/PYnFw6NmvALRd0qnoHkRikoBCpQfO/KOSTzXu4f3gP6qVoNCHxSUEhUkPKypzH5+bSpWVDbhyQEXQ5IqdNQSFSQ97+7Cs+27qP8SOySE3WS03il357RWpAaZkzcV4u3Vunc32/jlXvIBLDFBQiNWDWqq3kbj/AgyN7kpxU0Q2YReKHgkKkmpWUljFp3jrOateYa/q0D7ockTOmoBCpZn/5aAsbdx7koZE9SdJoQuoABYVINSoqKWPyO+s4N6MpI3u3DbockWqhoBCpRq+u3EzB7sM8OLInZhpNSN2goBCpJkeKS3lqfh4DujRnWE9Nzyt1h4JCpJq89MEmtu09wgSNJqSOUVCIVIPDRaVMeXc9F3VrycU9WgVdjki1UlCIVIMX3s9n54GjTLiiZ9CliFQ7BYXIGTpwtIRn3lvP0J6tyc5sEXQ5ItVOQSFyhv6wZCO7DxUzYaRGE1I3KShEzsDew8VMXbiBy89uy3mdmgVdjkiNUFCInIHnF21g35ESHtJoQuowBYXIadp1sIjnF2/k2r7t6d2hSdDliNQYBYXIafrdwvUcKi7lgcuzgi5FpEYpKEROQ+H+I0xfms8N/TqS1bZx0OWI1CgFhchpeHrBeopLnfEjNJqQuk9BIXKKtu09zB+Xb+LG8zuS2So96HJEapyCQuQUPTU/D3fn/uEaTUhiUFCInILNuw7xSs5mbrmgE51aNAy6HJFaoaAQOQVPzl+HmTH2Mo0mJHEoKESitHHnQf784RZuv7AL7ZrWD7ockVoTVVCY2VVmttbM8szs4Ura3WRmbmbZ4eVUM5tuZp+a2Roze6Rc+2Qz+8jMZlVwrCfN7MCpdkikpkyel0tachL3DusedCkitarKoDCzZGAKcDXQG7jVzHpX0K4xMA5YHrH6ZqCeu/cFBgD3mFlmxPbxwJoKjpUN6MY5EjNyt+/nr59s5Y6LM2nduF7Q5YjUqmhGFAOBPHff4O5FwAzg+gra/QL4NXAkYp0D6WaWAjQAioB9AGaWAVwLPBd5kHAw/Tfws1PrikjNmTQvl/S0FO4Z2i3oUkRqXTRB0RHYHLFcEF53nJn1Bzq5e/lLSK8BB4FtwCbgMXffFd42iVAYlJXbZyww0923RdUDkRr22da9vPnpV9x1SVeap6cFXY5IrUuJok1Fk//68Y1mScBE4M4K2g0ESoEOQHNgkZnNI3QJq9DdV5rZsIhjdSB0uWrYNw9VriizMcAYgM6dO0fRDZHTM3FuLk3qp/CDS7oGXYpIIKIZURQAnSKWM4CtEcuNgT7AAjPLBwYBM8PvM9wGzHb3YncvBJYA2cBgYFS4/QxguJm9CPQHegB54W0NzSyvoqLcfaq7Z7t7duvWraPtr8gp+XjzHuatKWTM0G40bZAadDkigYgmKFYAWWbW1czSgNHAzGMb3X2vu7dy90x3zwSWAaPcPYfQ5abhFpJOKES+cPdH3D0j3H40MN/db3f3v7t7u4hjHXL3HtXZYZFT8fjcXFqkp3HnYI0mJHFVGRTuXkLofYO3CX1C6RV3/8zMHjWzUVXsPgVoBKwmFDjT3H3VGdYsUitW5O9iYe4OfnRpNxrVi+YqrUjdZO5edasYl52d7Tk5OUGXIXWIuzN66jI27DzIwp9eRoO05KBLEql2ZrbS3bOraqdvZotUYOn6r1m+cRf3DeuukJCEp6AQKcfd+c2ctbRvWp/RA/WJOhEFhUg5C3J38OGmPdw/PIv6qRpNiCgoRCK4O4/PyaVTiwbcnJ0RdDkiMUFBIRJhzufb+XTLXsYNzyI1WS8PEVBQiBxXVhYaTXRrlc4/9e9Y9Q4iCUJBIRL290+3sXb7fsZfnkWKRhMix+nVIAKUlJYxcV4uvdo25lvndgi6HJGYoqAQAf768VY27DjIgyOzSEqq6D6YIolLQSEJr7i0jMnvrOOcDk248px2QZcjEnMUFJLwXltZwKZdh5hwRU/MNJoQKU9BIQntaEkpT76zjn6dmnFZrzZBlyMSkxQUktBeXrGZrXuP8JMremk0IXISCgpJWEeKS3lqfh4Du7ZgcI+WQZcjErMUFJKwXlz2JYX7jzJhpN6bEKmMgkIS0sGjJfx2wXqGZLXiwm4aTYhURkEhCekPS/PZdbCIh0b2DLoUkZinoJCEs+9IMVMXbmDEWW3o37l50OWIxDwFhSSc5xdtZO/hYh7UaEIkKgoKSSh7DhXx+8UbueqcdvTp2DTockTigoJCEsrUhRs4UFSi0YTIKVBQSMLYeeAo05bk861zO9CrXeOgyxGJGwoKSRjPLFjP0ZJSxl+eFXQpInFFQSEJYfu+I/zPsi/59vkZdG/dKOhyROKKgkISwpR38ygtc8aP0GhC5FQpKKTOK9h9iJc+2MR3LuhEpxYNgy5HJO4oKKTOe2p+HoYx9rIeQZciEpcUFFKn5e88yKsrC7jtws50aNYg6HJE4pKCQuq0J95ZR2qy8eNh3YMuRSRuKSikzsorPMAbH2/hexdl0qZJ/aDLEYlbCgqpsybNy6VBajL3DO0WdCkicU1BIXXSmm37mLVqG98f3JWWjeoFXY5IXIsqKMzsKjNba2Z5ZvZwJe1uMjM3s+zwcqqZTTezT81sjZk9Uq59spl9ZGazItb9Mfxcq83s92aWerqdk8Q1cW4ujeun8MMhGk2InKkqg8LMkoEpwNVAb+BWM+tdQbvGwDhgecTqm4F67t4XGADcY2aZEdvHA2vKHeqPwFlAX6ABcHeUfREBYFXBHuZ8vp0fDulG04b6O0PkTEUzohgI5Ln7BncvAmYA11fQ7hfAr4EjEescSDezFEL/6RcB+wDMLAO4Fngu8iDu/qaHAR8AGafWJUl0j8/NpVnDVL4/ODPoUkTqhGiCoiOwOWK5ILzuODPrD3Ry91mc6DXgILAN2AQ85u67wtsmAT8Dyip60vAlp+8Cs6OoUQSAlV/uYsHaHfzo0u40rq/RhEh1SImijVWwzo9vNEsCJgJ3VtBuIFAKdACaA4vMbB6hS1iF7r7SzIad5Hl/Cyx090UVFmU2BhgD0Llz5yi6IYngN3NyadUoje9d1CXoUkTqjGhGFAVAp4jlDGBrxHJjoA+wwMzygUHAzPAb2rcBs9292N0LgSVANjAYGBVuPwMYbmYvHjugmf0b0Bp46GRFuftUd8929+zWrVtH0Q2p65au38nS9V9z77AeNEyL5m8gEYlGNEGxAsgys65mlgaMBmYe2+jue929lbtnunsmsAwY5e45hC43DbeQdEIh8oW7P+LuGeH2o4H57n47gJndDVwJ3OruFV6WEinP3Xl8Ti7tmtTnny/UCFOkOlUZFO5eAowF3ib0CaVX3P0zM3vUzEZVsfsUoBGwmlDgTHP3VVXs8wzQFnjfzD42s59XVaPIwnU7yflyN/cN70H91OSgyxGpUyz04aL4lp2d7Tk5OUGXIQFxd25CP9GPAAAJaklEQVSYsoSdB4p49yfDSEvR90hFomFmK909u6p2ekVJ3Ju3ppBPCvYyfkSWQkKkBuhVJXGtrMx5fG4umS0b8u3zO1a9g4icMgWFxLXZn33Fmm37eODynqQk69dZpCbolSVxqzQ8mshq04hvndch6HJE6iwFhcStv32ylbzCAzxweU+Skyr6XqiIVAcFhcSlktIyJs3L5ez2Tbi6T7ugyxGp0xQUEpde/3AL+V8f4qGRPUnSaEKkRikoJO4UlZQx+Z11nJfRlMvPbhN0OSJ1noJC4s7LOZvZsucwD13RCzONJkRqmoJC4sqR4lKemr+O7C7NGZrVKuhyRBKCgkLiyh+Xb2L7vqNM0GhCpNYoKCRuHCoq4ekFeVzcvSUXdW8ZdDkiCUNBIXHjhfe/ZOeBIiZc0TPoUkQSioJC4sL+I8U88956hvVqzYAuLYIuRyShKCgkLkxbks+eQ8U8NFKjCZHapqCQmLf3UDHPLtrAFb3bcm5Gs6DLEUk4CgqJec8u2sD+IyU8qNGESCAUFBLTvj5wlGlLNnLdue05u32ToMsRSUgKColpv1u4gcPFpTxwuUYTIkFRUEjMKtx3hBfez+eGfh3p0aZR0OWIJCwFhcSs3y5YT3GpM/7yrKBLEUloCgqJSVv3HOZPyzdx84AMurRMD7ockYSWEnQBQXrh/XzWFx4gLSUp9C85+R+PU5Kol5wUsS2J1PDPtJQk6kWsj9wnLTn0T3MknJmn3s3DccYO7xF0KSIJL6GDYkX+bhbm7qCopIyi0jJKy7zajp2abCeESGryP4KkXvlgSUkiLSW5whBKLRdE5cOrfEjVK/98Eevj5SZ6m74+xCsrNnPbhZ3JaN4w6HJEEl5CB8WTt/Y/Ybm0zEOhUVLG0dLS44+LSsv+8bikjKMRy8WlJ7Y5epJ9TtheWkZRSSmHi0rZV1pSwf6lx5erMbu+EV7/CJvk4yGUmmJVhldFQVXvWDhVEl4V7V9ReD0xfx3JScZ9l2k0IRILEjooyktOMhqkJdMgLRlIDbocIDQ3dGToHC0XQseC6uhJgunY46MnbCuN2N+/EV57S4u/cazIAPRqDK/QqMlOCJctuw9z1+CutG1Sv/qeSEROm4IixqUkJ5GSnETDtKArCXF3SiJGXsWl3wyvyoItcrQUauvfCK+BmS01mhCJIQoKOSVmRmqykZqcRHq9oKsRkdqgj8eKiEilFBQiIlIpBYWIiFRKQSEiIpWKKijM7CozW2tmeWb2cCXtbjIzN7Ps8HKqmU03s0/NbI2ZPVKufbKZfWRmsyLWdTWz5Wa2zsxeNrMY+byPiEhiqjIozCwZmAJcDfQGbjWz3hW0awyMA5ZHrL4ZqOfufYEBwD1mlhmxfTywptyhfgVMdPcsYDfwg2g7IyIi1S+aEcVAIM/dN7h7ETADuL6Cdr8Afg0ciVjnQLqZpQANgCJgH4CZZQDXAs8da2yhr+kOB14Lr5oO3HAqHRIRkeoVTVB0BDZHLBeE1x1nZv2BTu4+ixO9BhwEtgGbgMfcfVd42yTgZ0BZRPuWwB53LznZc4mISO2K5gt3Fd1J7vhNHMwsCZgI3FlBu4FAKdABaA4sMrN5hC5hFbr7SjMbFu1znVCU2RhgTHjxgJmtrbwbJ9UK2Hma+8aautKXutIPUF9iVV3py5n2o0s0jaIJigKgU8RyBrA1Yrkx0AdYEL7BWztgppmNAm4DZrt7MVBoZkuAbKA/MMrMrgHqA03M7EXgu0AzM0sJjyrKP9dx7j4VmBpNJytjZjnunn2mx4kFdaUvdaUfoL7EqrrSl9rqRzSXnlYAWeFPI6UBo4GZxza6+153b+Xume6eCSwDRrl7DqHLTcMtJB0YBHzh7o+4e0a4/Whgvrvf7u4OvAvcFD78HcBfq6erIiJyOqoMivBf9mOBtwl9QukVd//MzB4NjxoqMwVoBKwmFDjT3H1VFfv8C/CQmeURes/i+apqFBGRmhPVTQHd/U3gzXLrfn6StsMiHh8g9BHZyo69AFgQsbyB0HsbteWML1/FkLrSl7rSD1BfYlVd6Uut9MO8OicXEBGROke38BARkUolXFCYWX74liIfm1lOeF0LM5sbvm3IXDNrHnSdVTlJP/7dzLaE130c/lRZzDOzZmb2mpl9Eb7Vy0XxeE7gpH2Jq/NiZr0iav3YzPaZ2QPxeE4q6UtcnZNjzOxBM/vMzFab2UtmVr82bnuUcJeezCwfyHb3nRHrfg3scvdfWuheVs3d/V+CqjEaJ+nHvwMH3P2xoOo6HWY2HVjk7s+Ff8kbAv+LODsncNK+PEAcnhc4fgufLcCFwH3E4Tk5plxfvk+cnRMz6wgsBnq7+2Eze4XQe8fXAK+7+wwzewb4xN2frs7nTrgRxUlcT+h2IaDbhtQqM2sCDCX86TZ3L3L3PcThOamkL/FsBLDe3b8kDs9JOZF9iVcpQIPwbZEaErrrRY3f9igRg8KBOWa2MvztboC27r4NIPyzTWDVRa+ifgCMNbNVZvb7eLg0AHQDdgDTLHQn4efC37mJx3Nysr5A/J2XY0YDL4Ufx+M5iRTZF4izc+LuW4DHCH0/bRuwF1hJLdz2KBGDYrC7n0/obrj3mdnQoAs6TRX142mgO9CP0C/SbwKsL1opwPnA0+7en9C9wU56K/sYd7K+xON5IXzpbBTwatC1nKkK+hJ35yQcZtcDXQndFimd0Ou/vGp/PyHhgsLdt4Z/FgJ/IfSdje1m1h4g/LMwuAqjU1E/3H27u5e6exnwLLX7fZTTVQAUuPux29O/Rug/27g7J5ykL3F6XiD0n9CH7r49vByP5+SYE/oSp+fkcmCju+8I3xbpdeBiwrc9Crc56W2PzkRCBYWZpVto3gzClwSuIPSt8ZmEbhcCcXDbkJP149iLOOyfCPUtprn7V8BmM+sVXjUC+Jw4Oydw8r7E43kJu5UTL9XE3TmJcEJf4vScbAIGmVlDMzP+8Vqp8dseJdSnnsysG6G/viF0meBP7v6fZtYSeAXoTOhk3BxxO/SYU0k//ofQUNqBfOCeY9eUY5mZ9SM0L0kasIHQJ1KSiKNzcsxJ+vIEcXZezKwhoekFurn73vC6uHqdHHOSvsTra+U/gFuAEuAj4G5C70nMAFqE193u7ker9XkTKShEROTUJdSlJxEROXUKChERqZSCQkREKqWgEBGRSikoRESkUgoKERGplIJCREQqpaAQEZFK/X8+qsaF+8aI2gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([-0.48406091, -0.48404742, -0.48501795, -0.48466498])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)\n",
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid_search.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid_search.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid_search.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "x_axis = min_child_samples_s\n",
    "\n",
    "plt.plot(x_axis, -test_means)\n",
    "#plt.errorbar(x_axis, -test_scores, yerr=test_stds ,label = ' Test')\n",
    "#plt.errorbar(x_axis, -train_scores, yerr=train_stds,label =  +' Train')\n",
    "\n",
    "plt.show()\n",
    "\n",
    "\n",
    "test_means"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 'min_child_samples': 60"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 5 candidates, totalling 15 fits\n",
      "[CV] sub_samples=0.5 .................................................\n",
      "[CV] sub_samples=0.5 .................................................\n",
      "[CV] sub_samples=0.5 .................................................\n",
      "[CV] sub_samples=0.6 .................................................\n",
      "[CV] ....... sub_samples=0.5, score=-0.4882779483756968, total= 2.1min\n",
      "[CV] sub_samples=0.6 .................................................\n",
      "[CV] ....... sub_samples=0.5, score=-0.4760216409099392, total= 2.1min\n",
      "[CV] sub_samples=0.6 .................................................\n",
      "[CV] ...... sub_samples=0.5, score=-0.48784386089611137, total= 2.2min\n",
      "[CV] sub_samples=0.7 .................................................\n",
      "[CV] ....... sub_samples=0.6, score=-0.4760216409099392, total= 2.3min\n",
      "[CV] sub_samples=0.7 .................................................\n",
      "[CV] ...... sub_samples=0.6, score=-0.48784386089611137, total= 2.0min\n",
      "[CV] sub_samples=0.7 .................................................\n",
      "[CV] ....... sub_samples=0.6, score=-0.4882779483756968, total= 2.0min\n",
      "[CV] sub_samples=0.8 .................................................\n",
      "[CV] ...... sub_samples=0.7, score=-0.48784386089611137, total= 2.1min\n",
      "[CV] sub_samples=0.8 .................................................\n",
      "[CV] ....... sub_samples=0.7, score=-0.4760216409099392, total= 2.4min\n",
      "[CV] sub_samples=0.8 .................................................\n",
      "[CV] ....... sub_samples=0.7, score=-0.4882779483756968, total= 2.2min\n",
      "[CV] sub_samples=0.9 .................................................\n",
      "[CV] ....... sub_samples=0.8, score=-0.4760216409099392, total= 2.2min\n",
      "[CV] sub_samples=0.9 .................................................\n",
      "[CV] ...... sub_samples=0.8, score=-0.48784386089611137, total= 2.2min\n",
      "[CV] sub_samples=0.9 .................................................\n",
      "[CV] ....... sub_samples=0.8, score=-0.4882779483756968, total= 2.2min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  12 out of  15 | elapsed:  7.2min remaining:  1.8min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ....... sub_samples=0.9, score=-0.4760216409099392, total= 1.6min\n",
      "[CV] ...... sub_samples=0.9, score=-0.48784386089611137, total= 1.7min\n",
      "[CV] ....... sub_samples=0.9, score=-0.4882779483756968, total= 1.6min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  15 out of  15 | elapsed:  8.5min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=0.7, importance_type='split', learning_rate=0.1,\n",
       "        max_bin=127, max_depth=-1, maxdepth=7, min_child_samples=60,\n",
       "        min_child_weight=0.001, min_split_gain=0.0, n_estimators=157,\n",
       "        n_jobs=-1, num_leaves=60, objective='multiclass',\n",
       "        random_state=None, reg_alpha=0.0, reg_lambda=0.0, silent=False,\n",
       "        subsample=0.7, subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'sub_samples': [0.5, 0.6, 0.7, 0.8, 0.9]},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#行采样参数\n",
    "#sub_samples\n",
    "params={'boosting_type':'gbdt',\n",
    "       'objective':'multiclass',\n",
    "       'n_jobs':-1,\n",
    "       'learning_rate':0.1,\n",
    "       'n_estimators':n_estimators_1,\n",
    "        'min_child_samples': 60,\n",
    "       'num_leaves':60,\n",
    "        'maxdepth':7,\n",
    "       'max_bin':127,\n",
    "       'subsample':0.7,\n",
    "       'bagging_freq':1,\n",
    "       'colsample_bytree':0.7\n",
    "       }\n",
    "lg=LGBMClassifier(silent=False,**params)\n",
    "sub_samples=[i/10.0 for i in range(5,10)]\n",
    "tuned_parameters=dict(sub_samples =sub_samples)\n",
    "\n",
    "grid_search=GridSearchCV(lg,n_jobs=-1,param_grid=tuned_parameters,cv=kfold,scoring=\"neg_log_loss\",verbose=5,refit=False)\n",
    "grid_search.fit(X_train,y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4840474205829825\n",
      "{'sub_samples': 0.5}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/liu/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/liu/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAEPVJREFUeJzt3W+MXFd9xvHvE7uBAqYEvEiQOKypwh8DUdJMU1S1gKgCBgm7lNA6UJS0QFClEKkNqoKQCnVUtaJV0xeNVAUUGtFCoHnTJQUsoARVlCCPIX9wUoNrQrMxEksCqVoaguHXF3sdhs3ac2e9u2PnfD/SyPeee87c354dP3P33p29qSokSW04bdoFSJLWj6EvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDeoV+ku1JDiQ5mOTqZbZflmQhye3d420j2z6d5PtJblnNwiVJk9s4rkOSDcB1wEXAPLA3yVxV3b2k68eq6oplnuIvgScB7+hT0ObNm2t2drZPV0lSZ9++fd+tqplx/caGPnAhcLCqDgEkuQnYCSwN/WVV1eeSvKJPX4DZ2VmGw2Hf7pIkIMm3+vTrc3rnTOC+kfX5rm2pNyS5M8nNSbb02bkkaX31Cf0s07b0r7R9ApitqnOBzwI3TlJEksuTDJMMFxYWJhkqSZpAn9CfB0aP3M8CDo92qKoHquqH3eoHgAsmKaKqrq+qQVUNZmbGnpKSJK1Qn9DfC5yTZGuS04FdwNxohyTPGlndAdyzeiVKklbL2Au5VXUkyRXAHmADcENV7U+yGxhW1RxwZZIdwBHgQeCyo+OT/BvwAuApSeaBt1bVntX/UiRJ4+Rku4nKYDAof3tHkiaTZF9VDcb18xO5ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIb0Cv0k25McSHIwydXLbL8syUKS27vH20a2XZrkG93j0tUsXpI0mY3jOiTZAFwHXATMA3uTzFXV3Uu6fqyqrlgy9unAe4EBUMC+buz3VqV6SdJE+hzpXwgcrKpDVfUIcBOws+fzvxr4TFU92AX9Z4DtKytVknSi+oT+mcB9I+vzXdtSb0hyZ5Kbk2yZcKwkaR30Cf0s01ZL1j8BzFbVucBngRsnGEuSy5MMkwwXFhZ6lCRJWok+oT8PbBlZPws4PNqhqh6oqh92qx8ALug7tht/fVUNqmowMzPTt3ZJ0oT6hP5e4JwkW5OcDuwC5kY7JHnWyOoO4J5ueQ/wqiRnJDkDeFXXJkmagrG/vVNVR5JcwWJYbwBuqKr9SXYDw6qaA65MsgM4AjwIXNaNfTDJNSy+cQDsrqoH1+DrkCT1kKrHnGKfqsFgUMPhcNplSNIpJcm+qhqM6+cnciWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kN6RX6SbYnOZDkYJKrj9Pv4iSVZNCtn57kQ0nuSnJHklesUt2SpBXYOK5Dkg3AdcBFwDywN8lcVd29pN8m4ErgyyPNbweoqpckeSbwqSS/XFU/Wa0vQJLUX58j/QuBg1V1qKoeAW4Cdi7T7xrg/cDDI23bgM8BVNV3gO8DgxOqWJK0Yn1C/0zgvpH1+a7tUUnOB7ZU1S1Lxt4B7EyyMclW4AJgywnUK0k6AWNP7wBZpq0e3ZicBlwLXLZMvxuAFwJD4FvAvwNHHrOD5HLgcoCzzz67R0mSpJXoc6Q/z88enZ8FHB5Z3wS8GLg1yb3AS4G5JIOqOlJVf1hV51XVTuBpwDeW7qCqrq+qQVUNZmZmVvq1SJLG6BP6e4FzkmxNcjqwC5g7urGqHqqqzVU1W1WzwG3AjqoaJnlSkicDJLkIOLL0ArAkaf2MPb1TVUeSXAHsATYAN1TV/iS7gWFVzR1n+DOBPUl+AtwPvGU1ipYkrUyfc/pU1SeBTy5p+5Nj9H3FyPK9wPNXXp4kaTX5iVxJaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ3pdbvEU8WffmI/dx/+72mXIUkrsu3ZT+W9r3vRmu7DI31Jasjj6kh/rd8hJelU55G+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUkF6hn2R7kgNJDia5+jj9Lk5SSQbd+s8luTHJXUnuSfLu1SpckjS5saGfZANwHfAaYBtwSZJty/TbBFwJfHmk+Y3AE6rqJcAFwDuSzJ542ZKklehzpH8hcLCqDlXVI8BNwM5l+l0DvB94eKStgCcn2Qj8PPAI4B+8l6Qp6RP6ZwL3jazPd22PSnI+sKWqblky9mbgf4FvA/8F/FVVPbjyciVJJ6JP6GeZtnp0Y3IacC1w1TL9LgR+DDwb2ApcleS5j9lBcnmSYZLhwsJCr8IlSZPrE/rzwJaR9bOAwyPrm4AXA7cmuRd4KTDXXcx9E/DpqvpRVX0H+CIwWLqDqrq+qgZVNZiZmVnZVyJJGqtP6O8FzkmyNcnpwC5g7ujGqnqoqjZX1WxVzQK3ATuqasjiKZ1XZtGTWXxD+I9V/yokSb2MDf2qOgJcAewB7gE+XlX7k+xOsmPM8OuApwBfY/HN40NVdecJ1ixJWqFU1fhe62gwGNRwOJx2GZJ0Skmyr6oec/p8KT+RK0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWpIr9BPsj3JgSQHk1x9nH4XJ6kkg279zUluH3n8JMl5q1W8JGkyY0M/yQbgOuA1wDbgkiTblum3CbgS+PLRtqr6x6o6r6rOA94C3FtVt69W8ZKkyfQ50r8QOFhVh6rqEeAmYOcy/a4B3g88fIznuQT46IqqlCStij6hfyZw38j6fNf2qCTnA1uq6pbjPM/vcIzQT3J5kmGS4cLCQo+SJEkr0Sf0s0xbPboxOQ24FrjqmE+Q/Arwg6r62nLbq+r6qhpU1WBmZqZHSZKklegT+vPAlpH1s4DDI+ubgBcDtya5F3gpMHf0Ym5nF57akaSp29ijz17gnCRbgftZDPA3Hd1YVQ8Bm4+uJ7kVeFdVDbv104A3Ai9bvbIlSSsx9ki/qo4AVwB7gHuAj1fV/iS7k+zosY+XAfNVdejESpUknahU1fhe62gwGNRwOJx2GZJ0Skmyr6oG4/r5iVxJaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQ3qFfpLtSQ4kOZjk6uP0uzhJJRmMtJ2b5EtJ9ie5K8kTV6NwSdLkNo7rkGQDcB1wETAP7E0yV1V3L+m3CbgS+PJI20bgH4C3VNUdSZ4B/GgV65ckTaDPkf6FwMGqOlRVjwA3ATuX6XcN8H7g4ZG2VwF3VtUdAFX1QFX9+ARrliStUJ/QPxO4b2R9vmt7VJLzgS1VdcuSsc8DKsmeJF9J8sfL7SDJ5UmGSYYLCwsTlC9JmkSf0M8ybfXoxuQ04FrgqmX6bQR+DXhz9+/rk/zGY56s6vqqGlTVYGZmplfhkqTJ9Qn9eWDLyPpZwOGR9U3Ai4Fbk9wLvBSY6y7mzgNfqKrvVtUPgE8Cv7QahUuSJtcn9PcC5yTZmuR0YBcwd3RjVT1UVZuraraqZoHbgB1VNQT2AOcmeVJ3UfflwN2P3YUkaT2MDf2qOgJcwWKA3wN8vKr2J9mdZMeYsd8D/prFN47bga9U1b+ceNmSpJVIVY3vtY4Gg0ENh8NplyFJp5Qk+6pqMK6fn8iVpIYY+pLUEENfkhpi6EtSQwx9SWrISffbO0kWgG+dwFNsBr67SuWsJuuajHVNxrom83is6zlVNfZPGpx0oX+ikgz7/NrSerOuyVjXZKxrMi3X5ekdSWqIoS9JDXk8hv710y7gGKxrMtY1GeuaTLN1Pe7O6UuSju3xeKQvSTqGUyb0x92cPcllSRaS3N493jay7dIk3+gel55Edf14pH1u6di1rKvr89tJ7u5uWv+RkfapzdeYuqY2X0muHdn315N8f2TbNF9fx6trzearZ21nJ/l8kq8muTPJa0e2vbsbdyDJq0+GupLMJvm/kTn7u3Wu6zlJPtfVdGuSs0a2rd5rrKpO+gewAfhP4LnA6cAdwLYlfS4D/naZsU8HDnX/ntEtnzHturpt/zPF+ToH+OrRuQCeeZLM17J1TXu+lvR/J3DDyTBfx6prLedrgu/l9cAfdMvbgHtHlu8AngBs7Z5nw0lQ1yzwtSnO1z8Bl3bLrwQ+vBavsVPlSL/vzdmX82rgM1X1YC3+ff/PANtPgrrWUp+63g5c180JVfWdrn3a83WsutbSpN/HS4CPdsvTnq9j1bXW+tRWwFO75V/gp3fc2wncVFU/rKpvAge755t2XWupT13bgM91y58f2b6qr7FTJfTH3py984buR6Obkxy9xWPfsetdF8ATs3hD+NuS/OYq1dS3rucBz0vyxW7/2ycYO426YLrzBSz+CM7i0em/Tjp2neuCtZuvvrW9D/jdJPMs3ir1nROMnUZdAFu70z5fSPLrq1RT37ruAN7QLb8e2JTkGT3H9naqhP5xb87e+QQwW1XnAp8Fbpxg7DTqAji7Fj999ybgb5L84jrWtZHFUymvYPEI8YNJntZz7DTqgunO11G7gJur6scrGDupE6kL1m6++tZ2CfD3VXUW8Frgw0lO6zl2GnV9m8U5Ox/4I+AjSZ7K6uhT17uAlyf5Kou3lr0fONJzbG+nSuiPuzk7VfVAVf2wW/0AcEHfsVOqi6o63P17CLgVOH+96ur6/HNV/aj7EfsAi2E71fk6Tl3Tnq+jdvGzp1CmPV/Hqmst56tvbW8FPt7V8CXgiSz+bZlpz9mydXWnmx7o2vexeA7+eetVV1Udrqrf6t503tO1PdTza+pvLS5arMFFkI0sXrzYyk8vgrxoSZ9njSy/Hrht5CLIN1m8AHJGt/z0k6CuM4AndMubgW9wnIt0a1DXduDGkf3fBzzjJJivY9U11fnq+j0fuJfu8y0nw+vrOHWt2XxN8L38FHBZt/xCFoMqwIv42Qu5h1i9C7knUtfM0TpYvOB6/zq/9jcDp3XLfwbsXovX2Kq8ANbjweKPYV9n8d33PV3bbmBHt/znwP5uMj8PvGBk7O+zeLHoIPB7J0NdwK8Cd3XtdwFvXee6wuJN6+/u9r/rJJmvZeua9nx16+8D/mKZsVObr2PVtdbz1fN7uQ34YlfD7cCrRsa+pxt3AHjNyVAXi+fTj/5f/QrwunWu62IW35y/DnyQ7k17tV9jfiJXkhpyqpzTlyStAkNfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SG/D8Lu7rNK9TLuwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([-0.48404742, -0.48404742, -0.48404742, -0.48404742, -0.48404742])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)\n",
    "\n",
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid_search.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid_search.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid_search.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "x_axis = sub_samples\n",
    "\n",
    "plt.plot(x_axis, -test_means)\n",
    "#plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = str(max_depths[i]) +' Test')\n",
    "#plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = str(max_depths[i]) +' Train')\n",
    "\n",
    "plt.show()\n",
    "\n",
    "test_means"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 'sub_samples': 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 5 candidates, totalling 15 fits\n",
      "[CV] colsample_bytree=0.5 ............................................\n",
      "[CV] colsample_bytree=0.5 ............................................\n",
      "[CV] colsample_bytree=0.5 ............................................\n",
      "[CV] colsample_bytree=0.6 ............................................\n",
      "[CV] . colsample_bytree=0.5, score=-0.49760119960384186, total= 1.8min\n",
      "[CV] colsample_bytree=0.6 ............................................\n",
      "[CV] .. colsample_bytree=0.6, score=-0.4796304339428399, total= 1.8min\n",
      "[CV] colsample_bytree=0.6 ............................................\n",
      "[CV] .. colsample_bytree=0.5, score=-0.4917788349507546, total= 1.9min\n",
      "[CV] colsample_bytree=0.7 ............................................\n",
      "[CV] . colsample_bytree=0.5, score=-0.48168706751344625, total= 1.9min\n",
      "[CV] colsample_bytree=0.7 ............................................\n",
      "[CV] .. colsample_bytree=0.6, score=-0.4953988679690641, total= 1.7min\n",
      "[CV] colsample_bytree=0.7 ............................................\n",
      "[CV] .. colsample_bytree=0.6, score=-0.4941771226170047, total= 1.8min\n",
      "[CV] colsample_bytree=0.8 ............................................\n",
      "[CV] .. colsample_bytree=0.7, score=-0.4790606654103015, total= 2.1min\n",
      "[CV] colsample_bytree=0.8 ............................................\n",
      "[CV] .. colsample_bytree=0.7, score=-0.4920759383045741, total= 2.2min\n",
      "[CV] colsample_bytree=0.8 ............................................\n",
      "[CV] . colsample_bytree=0.7, score=-0.49458365744444593, total= 1.9min\n",
      "[CV] colsample_bytree=0.9 ............................................\n",
      "[CV] . colsample_bytree=0.8, score=-0.49084590849735826, total= 1.9min\n",
      "[CV] colsample_bytree=0.9 ............................................\n",
      "[CV] .. colsample_bytree=0.8, score=-0.4817609861358693, total= 2.2min\n",
      "[CV] colsample_bytree=0.9 ............................................\n",
      "[CV] .. colsample_bytree=0.8, score=-0.4957111876238618, total= 1.9min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  12 out of  15 | elapsed:  6.4min remaining:  1.6min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] .. colsample_bytree=0.9, score=-0.4823690978750948, total= 1.5min\n",
      "[CV] ... colsample_bytree=0.9, score=-0.490800584416959, total= 1.3min\n",
      "[CV] . colsample_bytree=0.9, score=-0.49534626434452605, total= 1.4min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  15 out of  15 | elapsed:  7.7min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=LGBMClassifier(bagging_freq=1, boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=1.0, importance_type='split', learning_rate=0.1,\n",
       "        max_bin=127, max_depth=-1, maxdepth=7, min_child_samples=60,\n",
       "        min_child_weight=0.001, min_split_gain=0.0, n_estimators=157,\n",
       "        n_jobs=-1, num_leaves=60, objective='multiclass',\n",
       "        random_state=None, reg_alpha=0.0, reg_lambda=0.0, silent=False,\n",
       "        subsample=0.5, subsample_for_bin=200000, subsample_freq=0),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid={'colsample_bytree': [0.5, 0.6, 0.7, 0.8, 0.9]},\n",
       "       pre_dispatch='2*n_jobs', refit=False, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#列采样参数\n",
    "#sub_feature\n",
    "params={'boosting_type':'gbdt',\n",
    "       'objective':'multiclass',\n",
    "       'n_jobs':-1,\n",
    "       'learning_rate':0.1,\n",
    "       'n_estimators':n_estimators_1,\n",
    "        'min_child_samples': 60,\n",
    "       'num_leaves':60,\n",
    "        \n",
    "        'maxdepth':7,\n",
    "       'max_bin':127,\n",
    "       'subsample':0.5,\n",
    "       'bagging_freq':1,\n",
    "      # 'colsample_bytree':0.7\n",
    "       }\n",
    "lg=LGBMClassifier(silent=False,**params)\n",
    "colsample_bytree=[i/10.0 for i in range(5,10)]\n",
    "tuned_parameters=dict(colsample_bytree = colsample_bytree)\n",
    "\n",
    "grid_search=GridSearchCV(lg,n_jobs=-1,param_grid=tuned_parameters,cv=kfold,scoring=\"neg_log_loss\",verbose=5,refit=False)\n",
    "grid_search.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.48857291865748886\n",
      "{'colsample_bytree': 0.7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/liu/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/liu/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAD8CAYAAABZ/vJZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3Xl4VOX5//H3nZ2EsCaRJYGEnbDDGJG6KxYRwQUUUGqrFaFSN/y61FZRq1Xr9lMRpC7FBdGi1YCKWi22yiIJa8Ia9gBC2MKalfv3x5zQAQKZhCRnktyv65orM888z5nPGYbcOeeZc46oKsYYY0x5BbkdwBhjTM1kBcQYY0yFWAExxhhTIVZAjDHGVIgVEGOMMRViBcQYY0yFWAExxhhTIVZAjDHGVIgVEGOMMRUS4naAqhQTE6OJiYluxzDGmBolPT19l6rGltWvVheQxMRE0tLS3I5hjDE1iohs8qef7cIyxhhTIVZAjDHGVIgVEGOMMRViBcQYY0yFWAExxhhTIVZAjDHGVIgVEGOMMRViBaQUeYXFTEjNZN/hArejGGNMwLICUorlW3OZtmAzQyfPY9u+I27HMcaYgGQFpBRnJzbhnVtT2JGbx3WT5rJmxwG3IxljTMCxAnIKfds05cPbz6XoqDJs8jzSN+1xO5IxxgQUKyCnkdyiAZ+M7UeTqDBG/m0B/1qxw+1IxhgTMKyAlCGhSSQzxpxLp2bR3P5eOh8u3Ox2JGOMCQhWQPzQtH44027ryy/axfDAx8t59bu1qKrbsYwxxlVWQPwUFR7CG7/ycE2vljz39RompGZSfNSKiDGm7qrV1wOpbGEhQTw/rAex0eFM+c96dh0s4IUbehAeEux2NGOMqXZ+bYGIyAARWS0iWSLy4Gn6DRURFRGP8zhMRN4WkeUislRELvLp28dpzxKRl0VEnPa/isgqEVkmIv8UkUZOe6KIHBGRJc5t8hmteQUFBQl/GNiZhwd25vPl2/nN2ws5kFfoRhRjjHFVmQVERIKBicAVQDIwQkSSS+kXDdwJLPBpvg1AVbsB/YHnRaTkNScBo4H2zm2A0/4N0FVVuwNrgId8lrdOVXs6tzF+r2UVuO2CNrx4Qw9+2rCHG16fz84DeW7GMcaYaufPFkgKkKWq61W1AJgODCml3xPAs4Dvb9Jk4FsAVd0J7AM8ItIcaKCq89Q7G/0OcLXT72tVLXLGzwfiy79a1eOaXvG8cbOHjbsPcd2kuWzYdcjtSMYYU238KSAtgS0+j7OdtmNEpBeQoKqzThi7FBgiIiEikgT0ARKc8dmnW6bjFuBLn8dJIrJYRL4XkfP9yF7lLuoYx7Tb+nIov5ihk+ayLHuf25GMMaZa+FNApJS2Y18/cnZJvQiML6XfW3iLQxrwEjAXKCprmc5yH3b6vu80bQdaqWov4F5gmog0OCmsyGgRSRORtJycnDJWrXL0TGjEjDHnUi8smOFT5vPftdXzusYY4yZ/Ckg23q2GEvHANp/H0UBXYI6IbAT6Aqki4lHVIlW9x5mzGAI0AtY6y4w/1TJF5GZgEHCjs4sLVc1X1d3O/XRgHdDhxLCqOkVVParqiY2N9WP1Kkeb2Pp8MrYfrZpEcsvfF/LZkq3V9trGGOMGfwrIQqC9iCSJSBgwHEgteVJVc1U1RlUTVTUR77zFYFVNE5FIEYkCEJH+QJGqrlDV7cABEenrfPvqV8BnTr8BwAPOMg6XvI6IxDoT+ohIG7wT7+vP+B2oRHENIvhozLn0btWYu6Yv4c0fNrgdyRhjqkyZBcSZ0B4HfAWsBD5S1UwReVxEBpcxPA5YJCIr8RaFUT7PjQXeALLwbk2UzHW8iner5psTvq57AbBMRJYCM4AxqhpwZzhsEBHK1FtSuKJrM56YtYKnv1xlR60bY2olqc2/3Dwej6alpbny2sVHlUdTM3hv/mau7d2SZ67rTmiwHfhvjAl8IpKuqp6y+tmR6FUkOEh4YkhX4qIjeOGbNew5VMBrN/YmMszecmNM7WB/ElchEeHOS9vzl2u78Z81OYz82wL2HLLL5BpjagcrINVgREorJt3Uh5Xb9zN08lyy9x4ue5AxxgQ4KyDV5JddmvHureew60A+102ay6qf97sdyRhjzogVkGqUktSEf4zpB8CwyfNYsH63y4mMMabirIBUs47Novl4bD9io8MZ9dZPzM742e1IxhhTIVZAXBDfOJKPx/SjS4sG/O79dN5fsMntSMYYU25WQFzSOCqM9397Dhd1jOPhf2bw//5ll8k1xtQsVkBcFBkWwuuj+nBd73he/Nca/vhphl0m1xhTY9hRbS4LDQ7iuWHdiWsQzqQ569h9sICXhvckItQuk2uMCWy2BRIARIQHBnTiT4OSmZ35Mze/9RO5R+wyucaYwGYFJIDcel4SL4/oxaLNe7nh9Xns2G+XyTXGBC4rIAFmcI8WvP3rFLbsOcy1r81lXc5BtyMZY0yprIAEoPPaxzB99LnkF3kvk7tki10m1xgTeKyABKhu8Q2ZMaYf0RGhjJgynzmrd7odyRhjjmMFJIAlxkQxY+y5JMVE8dupaXyyKNvtSMYYc4wVkAAXFx3Bh7f3JSWpCfd+tJQp/1nndiRjjAGsgNQI0RGhvP2bs7mye3Oe+mIVf561gqN2wKExxmV2IGENER4SzCvDexFbP5w3ftjAroP5PDu0B2Eh9jeAMcYdfv32EZEBIrJaRLJE5MHT9BsqIioiHudxmIi8LSLLRWSpiFzk07eP054lIi+LiDjtTUTkGxFZ6/xs7LSL0y9LRJaJSO8zWvMaKChIePSqZP7vlx35dMk2bp26kEP5RW7HMsbUUWUWEBEJBiYCVwDJwAgRSS6lXzRwJ7DAp/k2AFXtBvQHnheRktecBIwG2ju3AU77g8C3qtoe+NZ5jPP6JX1HO+PrHBHhjovb8ex13Zm7bjcj/zaf3Qfz3Y5ljKmD/NkCSQGyVHW9qhYA04EhpfR7AngW8D18OhlvEUBVdwL7AI+INAcaqOo89Z6C9h3gamfMEGCqc3/qCe3vqNd8oJGznDrp+rMTeP2mPqz6+QBDJ89jyx67TK4xpnr5U0BaAlt8Hmc7bceISC8gQVVnnTB2KTBEREJEJAnoAyQ4432/k+q7zLNUdTuA8zPO3xx1zWXJZzHttnPYc6iAayfNZcU2u0yuMab6+FNApJS2Y18BcnZJvQiML6XfW3h/0acBLwFzgaKyllmRHD55RotImoik5eTklLHImq9P6ybMGHMuIUHCDa/PY946u0yuMaZ6+FNAsvFuNZSIB7b5PI4GugJzRGQj0BdIFRGPqhap6j2q2lNVhwCNgLXOMuNPscwdJbumnJ8lh2CXlQMAVZ2iqh5V9cTGxvqxejVf+7Oi+eR3/WjWMIKb3/qJL5ZvdzuSMaYO8KeALATai0iSiIQBw4HUkidVNVdVY1Q1UVUTgfnAYFVNE5FIEYkCEJH+QJGqrnB2TR0Qkb7Ot69+BXzmLDIVuNm5f/MJ7b9yvo3VF8gt2dVloHnDevxjzLl0j2/IHdMW8e68jW5HMsbUcmUWEFUtAsYBXwErgY9UNVNEHheRwWUMjwMWichK4AFglM9zY4E3gCxgHfCl0/400F9E1uL95tbTTvsXwHqn/9+A35W9enVLo8gw3r31HC7tFMefPsvkha9X22VyjTFVRmrzLxiPx6NpaWlux6h2RcVHefifGXyYtoURKQk8MaQrIcF2wKExxj8ikq6qnrL62ZHotVBIcBBPX9eNuAbhvPJdFjkHCnh1ZC+7TK4xplLZn6W1lIgw/vKOPD6kC9+u2sGoNxeQe9guk2uMqTxWQGq5X52byKsjerN0Sy7DXp/L9twjbkcyxtQSVkDqgCu7N+fvt5zNtn15XPfaXLJ2HnA7kjGmFrACUkf0axvD9NF9KShWhk6eR/qmvW5HMsbUcFZA6pCuLRvyydh+NKoXyo1vzOe7VTvcjmSMqcGsgNQxrZpGMmNsP9rHRXPbO+l8lLal7EHGGFMKKyB1UEz9cD4Y3Zd+bZty/4xlvDYnyw44NMaUmxWQOqp+eAhv3nw2Q3q24NnZq3lspl0m1xhTPnYgYR0WFhLEi9f3JKZ+OG/+sIHdhwp4blh3wkPsgENjTNmsgNRxQUHCH6/sTFx0OH/5chV7DxUweVQf6ofbR8MYc3q2C8sgItx+YVueH9aDeet3M3zKPHIO2GVyjTGnZwXEHHNdn3jeuNnDup2HGDp5Lpt2H3I7kjEmgFkBMce5uGMc0247h/1HCrlu0lwytua6HckYE6CsgJiT9GrVmBlj+xEeEswNr8/jx6xdbkcyxgQgKyCmVG1j6/Px2H7EN47k12//xMylJ1092BhTx1kBMafUrGEEH405l14Jjblz+mLe/nGD25GMMQHECog5rYb1Qnnn1hQuTz6Lx2au4JnZq+yodWMMYAXE+CEiNJjXbuzDyHNaMWnOOu6fsYyi4qNuxzLGuMyvAiIiA0RktYhkiciDp+k3VERURDzO41ARmSoiy0VkpYg85NP3LhHJEJFMEbnbp/1DEVni3DaKyBKnPVFEjvg8N7niq23KKzhIePLqrtx9WXv+kZ7N7e+mc6Sg2O1YxhgXlXm4sYgEAxOB/kA2sFBEUlV1xQn9ooE7gQU+zcOAcFXtJiKRwAoR+QCoD9wGpAAFwGwR+VxV16rqDT7LfB7w/R7pOlXtWZEVNWdORLj7sg7E1A/nkc8yuPGN+bx589k0jgpzO5oxxgX+bIGkAFmqul5VC4DpwJBS+j0BPAvk+bQpECUiIUA9vMViP9AZmK+qh1W1CPgeuMZ3YSIiwPXAB+VbJVPVburbmtdu7E3Gtv0Me30e2/bZZXKNqYv8KSAtAd+LRmQ7bceISC8gQVVnnTB2BnAI2A5sBp5T1T1ABnCBiDR1tkwGAgknjD0f2KGqa33akkRksYh8LyLnlxZWREaLSJqIpOXk5PixeqYiBnRtzju3pLAjN49rX5vLmh12mVxj6hp/CoiU0nbsazgiEgS8CIwvpV8KUAy0AJKA8SLSRlVXAs8A3wCzgaVA0QljR3D81sd2oJWq9gLuBaaJSIOTgqlOUVWPqnpiY2P9WD1TUX3bNOWjMedyVJWhk+aycvt+tyMZY6qRPwUkm+O3DuIB36PKooGuwBwR2Qj0BVKdifSRwGxVLVTVncCPgAdAVd9U1d6qegGwBzi2peHs8roW+LCkTVXzVXW3cz8dWAd0KN/qmsrWuXkDPh7bj7CQIP7wz+V2TRFj6hB/CshCoL2IJIlIGDAcSC15UlVzVTVGVRNVNRGYDwxW1TS8u60uEa8ovMVlFYCIxDk/W+EtFr5bG5cBq1Q1u6RBRGKdCX1EpA3QHlhfwfU2lSihSSQPDOjE4s37+GTxVrfjGGOqSZkFxJnkHgd8BawEPlLVTBF5XEQGlzF8It5vXGXgLURvq+oy57mPRWQFMBO4Q1X3+owbzsmT5xcAy0RkKd65lTHOfIoJANf1jqdXq0Y8/eUq9ucVuh3HGFMNpDYfVezxeDQtLc3tGHXG8uxcBk/8gVt/kcQfByW7HccYU0Eikq6qnrL62ZHoptJ0i2/I8LMT+Pvcjay1b2UZU+tZATGV6r7LOxIZFsyEmZl2zixjajkrIKZSNa0fzvjLO/Jj1m5mZ/zsdhxjTBWyAmIq3Y3ntKJTs2j+/PlKO1+WMbWYFRBT6UKCg5gwuAtb9x1h0vfr3I5jjKkiVkBMlejbpimDe7Rg8vfr2LLnsNtxjDFVwAqIqTJ/GNiZkCDhiVkryu5sjKlxrICYKtOsYQTjLmnH1yt28P0aO7GlMbWNFRBTpW49L4mkmCgeS82koMiuYmhMbWIFxFSp8JBgHhmUzPpdh3j7xw1uxzHGVCIrIKbKXdwpjks7xfHyt2vZsT+v7AHGmBrBCoipFn8alExhsfL0l6vcjmKMqSRWQEy1SIyJ4rYLkvjn4q2kbbSTKBtTG1gBMdXmjovb0bxhBI98lkmxXXjKmBrPCoipNpFhITx8ZWdWbN/PtJ82ux3HGHOGrICYanVlt+ac26Ypz3+9mr2HCtyOY4w5A1ZATLUSESYM7sKBvCKe+3q123GMMWfACoipdh2bRTOqb2um/bSZjK25bscxxlSQXwVERAaIyGoRyRKRB0/Tb6iIqIh4nMehIjJVRJaLyEoRecin710ikiEimSJyt0/7BBHZKiJLnNtAn+cecjKsFpFfVmyVTSC4p38HmkSGMSHVLjxlTE1VZgERkWBgInAFkAyMEJGTLngtItHAncACn+ZhQLiqdgP6ALeLSKKIdAVuA1KAHsAgEWnvM+5FVe3p3L5wlp8MDAe6AAOA15xspgZqWC+U+wd0JG3TXj5dstXtOMaYCvBnCyQFyFLV9apaAEwHhpTS7wngWcD3UGMFokQkBKgHFAD7gc7AfFU9rKpFwPfANWXkGAJMV9V8Vd0AZDnZTA01rE8CPeIb8tQXqziQV+h2HGNqPFVl76EC1uw4wIZdh6r89UL86NMS2OLzOBs4x7eDiPQCElR1lojc5/PUDLy/+LcDkcA9qrpHRDKAJ0WkKXAEGAik+YwbJyK/ctrGq+peJ8f8E3K09CO/CVBBQcJjQ7py9cQfeeW7LP4wsLPbkYwJSIXFR9l1MJ+d+/PZeSCfnAP57DyQ53M/n13O/YJi70lLB3Vvzqsje1dpLn8KiJTSdmyntYgEAS8Cvy6lXwpQDLQAGgP/FZF/qepKEXkG+AY4CCwFipwxk/Buzajz83nglrJy+OQZDYwGaNWqVdlrZ1zVM6ER13vieeuHDVzvSaBdXH23IxlTLVSVg/lFxwpAyc+dB/LIKXm8P5+cg/nsOcVX3htHhhIXHUFcg3DaxkQR2yCc2PrhxDWIoG1sVJWvgz8FJBtI8HkcD2zzeRwNdAXmiAhAMyBVRAYDI4HZqloI7BSRHwEPsF5V3wTeBBCRp5zXQVV3lCxYRP4GzPIzB874KcAUAI/HY7OzNcD9AzrxZcbPPDYzk3duScH5HBlTIxUfVfYcKjhuCyHHd6vBKQo79+dzpLD4pPGhwUJs/XBiG0SQ0CSSPomNiYsOJzY6nLjoCOdnODH1wwkLcfeLtP4UkIVAexFJArbincgeWfKkquYCMSWPRWQOcJ+qponIpcAlIvIe3l1YfYGXnH5xqrpTRFoB1wLnOu3NVXW7s7hrgAznfiowTURewLtF0x74qUJrbQJKTP1w7rmsA4/PWsHXK3bwyy7N3I5kzEnyCouPFYFjWwv7/1cYSorC7kMFpZ6qJzo8hNgG3l/+3eMb+RSF4wtDo8jQGvNHVJkFRFWLRGQc8BUQDLylqpki8jiQpqqppxk+EXgbbxEQ4G1VXeY897EzB1II3OHMcwA8KyI98e6e2gjc7uTIFJGPgBV4d3fdoaonl29TI406tzXTF27miVkruLBDLBGh9gU7U/VUldwjhcfPK+w/eXfSzgP5HMgrOml8kEDT+uHHikFy8wbHFQPfrYZ6YbXvMy21+Tv4Ho9H09LSyu5oAsLcdbsY+bcF3HNZB+66rH3ZA4w5Bd9J59KKge9upZJJZ18RoUHHFYLjisGxeYZwmkaFExxUM7YWykNE0lXVU1Y/f3ZhGVMt+rWN4cpuzXltThbX9m5JQpNItyOZAHMwv4id+312IflsOeT4TESXNekcGx1Om5goYksKQ4OIY0UhLjqc+uEhNWY3kpusgJiA8ocrO/Pdqp08+flKJo/q43YcEwAOFxQx9r1FLNy4h8MFp5l0jg4nvnEkvVs39ikGgTXpXNtYATEBpWWjetxxcVue+3oNP6zdxXntY8oeZGotVeXBj5fzn7U53HROa1o2rnfc7qS46HAa1gslqBbuRqoJrICYgPPb89vwUVo2E2Zm8uVd5xMabH811lVv/biR1KXbGN+/A7+/1ObFAo39zzQBJyI0mEcGJZO18yBT5250O45xybx1u3nqi5X0Tz6LOy5u53YcUworICYgXdo5jos6xvLSv9ay80Be2QNMrbJt3xHGTVtE66aRvHB9D9tFFaCsgJiAJCI8MiiZ/KJinvnSLjxVl+QVFjP2vXTyCouZMqoP0RGhbkcyp2AFxASsNrH1ufW8Nny8KJv0TXvLHmBqPFXl0c8yWZqdy/PX96RdXLTbkcxpWAExAe33l7TjrAbhTEjNLPX0EKZ2+eCnLXyYtoU7Lm7LgK52SptAZwXEBLSo8BD+MLAzy7fm8uHCLWUPMDXWos17eTQ1gws6xHJv/45uxzF+sAJiAt7gHi1ISWrCX79axb7DpR9hbGq2nQfyGPteOs0aRvDy8J618vQgtZEVEBPwRIQJV3Uh90ghL3yzxu04ppIVFh9l3PuLyT1SyOs3eWgUGeZ2JOMnKyCmRkhu0YCb+rbmvfmbWLFtv9txTCV68vOV/LRxD89c153kFg3cjmPKwQqIqTHu7d+BhvVCmZCaSW0+i3Rd8smibP4+dyO3/CKJIT3tCtU1jRUQU2M0igzj/37ZiZ827iF16UkXozQ1TMbWXB76ZDnnJDXhoYGd3I5jKsAKiKlRbjg7gW4tG/LUFys5lH/yBX5MzbD3UAFj3kuncWQYr47sbec7q6HsX83UKMFBwmNDurBjfz6vfJfldhxTAcVHlTunL2bn/nwmj+pDbHS425FMBVkBMTVO71aNua53PG/+sJ71OQfdjmPK6a9frea/a3fx+JAu9Exo5HYccwasgJga6YErOhIeEszjs1bYhHoN8uXy7Uz+fh0jUloxPKWV23HMGfKrgIjIABFZLSJZIvLgafoNFREVEY/zOFREporIchFZKSIP+fS9S0QyRCRTRO72af+riKwSkWUi8k8RaeS0J4rIERFZ4twmV3y1TU0XFx3B3Ze1Z87qHL5dudPtOMYPa3cc4L5/LKVnQiMmDE52O46pBGUWEBEJBiYCVwDJwAgROelfX0SigTuBBT7Nw4BwVe0G9AFudwpBV+A2IAXoAQwSkZKrxXwDdFXV7sAa4CGf5a1T1Z7ObUw519XUMjf3S6RdXH0en7WCvMKTL3VqAsf+vEJGv5tOvbBgJt3Um/CQYLcjmUrgzxZICpClqutVtQCYDgwppd8TwLOA78UbFIgSkRCgHlAA7Ac6A/NV9bCqFgHfA9cAqOrXThvAfCC+/Ktl6oLQ4CAmXNWFzXsO88Z/17sdx5zC0aPKvR8uYcuew0wc2ZvmDeu5HclUEn8KSEvA9yx22U7bMSLSC0hQ1VknjJ0BHAK2A5uB51R1D5ABXCAiTUUkEhgIJJTy2rcAX/o8ThKRxSLyvYic70d2U8ud1z6GAV2a8eq/s9i674jbcUwpXvkui3+t3MnDV3bmnDZN3Y5jKpE/BaS0s5odm7UUkSDgRWB8Kf1SgGKgBZAEjBeRNqq6EngG7+6q2cBS4Lgv9YvIw07b+07TdqCVqvYC7gWmichJ5z0QkdEikiYiaTk5OX6snqnp/jioM6rw1Ocr3Y5iTvDdqh289O0arunVkl/3S3Q7jqlk/hSQbI7fOogHfA8Djga6AnNEZCPQF0h1JtJHArNVtVBVdwI/Ah4AVX1TVXur6gXAHmBtyQJF5GZgEHCjOl+xUdV8Vd3t3E8H1gEdTgyrqlNU1aOqntjYWH/eA1PDxTeO5HcXtePz5duZm7XL7TjGsXHXIe6avoTOzRrw1DXdELEz7NY2/hSQhUB7EUkSkTBgOJBa8qSq5qpqjKomqmoi3nmLwaqahne31SXiFYW3uKwCEJE452cr4FrgA+fxAOABZxmHS15HRGKdCX1EpA3QHrAd3waA2y9sQ3zjekyYmUlh8VG349R5h/KLuP3ddIKDhNdH9aFemE2a10ZlFhBnQnsc8BWwEvhIVTNF5HERGVzG8IlAfbxzHguBt1V1mfPcxyKyApgJ3KGqJdcsfRXvVs03J3xd9wJgmYgsxTu3MsaZTzGGiNBg/jQomTU7DvLuvE1ux6nTVJX7P17G2p0HeHl4LxKaRLodyVQRqc0HYXk8Hk1LS3M7hqkmqsqv3vqJJVv28e/7LiKmvp0iww1T/rOOp75Yxf0DOvK7i9q5HcdUgIikq6qnrH52JLqpNUSER6/qwpGCYp6dvcrtOHXS3KxdPP3lKq7o2oyxF7Z1O46pYlZATK3SLq4+t5yXxEdp2SzZss/tOHXK1n1HGPfBYtrE1uevw3rYpHkdYAXE1Dq/v6QdcdHhPPpZBkeP1t5dtIEkr7CYMe+mU1h0lNdH9aF+eIjbkUw1sAJiap3oiFAeGtiJpdm5/CN9S9kDzBlRVf74aQbLt+bywg09aRtb3+1IpppYATG10tU9W+Jp3ZhnZ68m90ih23Fqtffmb2JGejZ3XtKO/slnuR3HVCMrIKZWEhEmDO7CnsMFvPjNGrfj1Frpm/bw2MwVXNwxlrsvO+m4XlPLWQExtVbXlg0ZmdKKd+dvYvXPB9yOU+vs3J/HmPcW0bJxPV66oRdBQTZpXtdYATG12n2XdyQ6IoRHUzPswlOVqKDoKGPfX8TBvCJeH9WHhpGhbkcyLrACYmq1xlFhjL+8I/PX7+Hz5dvdjlNrPDFrBemb9vLM0O50anbSOU1NHWEFxNR6I1Nakdy8AU9+vpLDBUVlDzCn9Y+0Lbw7fxO3nZ/E4B4t3I5jXGQFxNR6wUHC40O6sD03j4n/znI7To22PDuXhz/NoF/bpjwwoJPbcYzLrICYOsGT2IRrerXkb//ZwMZdh9yOUyPtPpjPmPfSia0fzisjehESbL8+6jr7BJg646ErOhEaLDwxa4XbUWqcouKj/P6DxeQczGfSTb1paieqNFgBMXVIXIMI7ry0Pd+u2sm/V+10O06N8uxXq5m7bjd/vror3eMbuR3HBAgrIKZO+c0vkmgTG8VjMzPJLyp2O06NMHPpNqb8Zz039W3F9Z6EsgeYOsMKiKlTwkKCePSqLmzcfZg3f9jgdpyAt/rnA9w/Yxl9WjfmkUFd3I5jAowVEFPnXNghlv7JZ/HKt1lszz3idpyAlXukkNvfTaN+RAiv3dibsBD7dWGOZ58IUyc9MiiZYlWe+sIuPFWao0eVu6cvJnvvESbd2JuzGkS4HckEICsgpk5KaBLJmAvbMnPpNuav3+12nIDz0refCUC8AAAT+klEQVRr+ffqHB69KhlPYhO345gA5VcBEZEBIrJaRLJE5MHT9BsqIioiHudxqIhMFZHlIrJSRB7y6XuXiGSISKaI3O3T3kREvhGRtc7Pxk67iMjLToZlItK74qttDIy9sC0tG9VjQmomRcVH3Y4TML5ZsYOXv13Ldb3jualva7fjmABWZgERkWBgInAFkAyMEJHkUvpFA3cCC3yahwHhqtoN6APcLiKJItIVuA1IAXoAg0SkvTPmQeBbVW0PfOs8xnn99s5tNDCpnOtqzHHqhQXzxys7s+rnA7y/YLPbcQLC+pyD3PvhErq2bMCT13S1y9Ka0/JnCyQFyFLV9apaAEwHhpTS7wngWSDPp02BKBEJAeoBBcB+oDMwX1UPq2oR8D1wjTNmCDDVuT8VuNqn/R31mg80EpHmfq6nMaUa0LUZv2jXlOe/Xs3ug/lux3HVwfwibn83ndCQICbf1IeI0GC3I5kA508BaQn4Xhc022k7RkR6AQmqOuuEsTOAQ8B2YDPwnKruATKAC0SkqYhEAgOBki+Yn6Wq2wGcn3H+5jCmvESECVd14XBBMc99vdrtOK5RVe6fsZR1OQd5dUQv4htHuh3J1AD+FJDStmGPXVhBRIKAF4HxpfRLAYqBFkASMF5E2qjqSuAZ4BtgNrAUKOs0qafN4ZNntIikiUhaTk5OGYs0BtqfFc3N/RKZvnALy7L3uR3HFZO/X88Xy3/mwSs60a9djNtxTA3hTwHJ5n9bBwDxwDafx9FAV2COiGwE+gKpzkT6SGC2qhaq6k7gR8ADoKpvqmpvVb0A2AOsdZa3o2TXlPOz5JwTZeXAWe4UVfWoqic2NtaP1TMG7rqsPU2jwnnks0yOHq1bF57679oc/vrVKq7s3pzbzm/jdhxTg/hTQBYC7UUkSUTCgOFAasmTqpqrqjGqmqiqicB8YLCqpuHdbXWJ8w2qKLzFZRWAiMQ5P1sB1wIfOItMBW527t8MfObT/itnWX2B3JJdXcacqQYRoTx4RSeWbNnHx4uy3Y5TbbbsOczvP1hM+7honr2uu02am3Ips4A4k9zjgK+AlcBHqpopIo+LyOAyhk8E6uOd81gIvK2qy5znPhaRFcBM4A5V3eu0Pw30F5G1QH/nMcAXwHogC/gb8Ds/19EYv1zbqyW9WjXimdmr2J9X6HacKpdXWMyY99IpPqq8PqoPUeEhbkcyNYzU5utEezweTUtLczuGqUGWZ+cyeOIP3PKLJP406KRvq9caqsr4j5byzyVbefNmD5d0OsvtSCaAiEi6qnrK6mdHohvjo1t8Q4afncDUuRtZu+OA23GqzNS5G/lk8VbuvrSDFQ9TYVZAjDnBfZd3JDIsmAkzM6mNW+g/bdjDnz9fyWWd4/j9Je3cjmNqMCsgxpygaf1wxl/ekR+zdjM742e341Sqn3Pz+N37i0hoEskLN/QkKMgmzU3FWQExphQ3ntOKTs2i+fPnKzlSUDsuPJVfVMzY99M5XFDE66P60CAi1O1IpoazAmJMKUKCg3hscBe27jvCpDlZbsepFI/NXMHizft4blgPOpwV7XYcUwtYATHmFM5p05TBPVow+T/r2bz7sNtxzsiHCzczbcFmxlzYloHd7BRypnJYATHmNP4wsDMhQcITn69wO0qFLdmyjz99msl57WL4v192dDuOqUWsgBhzGs0aRjDuknZ8s2IH36+peedW23Uwn7HvpRMbHc4rI3oRbJPmphJZATGmDLeel0RSTBSPpWZSUFRzLjxVVHyUcdMWsedQAa+P6kPjqDC3I5laxgqIMWUIDwnmkUHJrN91iLd/3OB2HL/95ctVzF+/h79c242uLRu6HcfUQlZAjPHDxZ3iuLRTHC9/u5Yd+/PKHuCyz5Zs5c0fNvDrfolc2zve7TimlrICYoyfHrkqmcJi5S9frHQ7ymmt2LafBz5eRkpiEx6+srPbcUwtZgXEGD+1bhrF6Ava8OmSbSzcuMftOKXad7iA299Lo2G9UF69sRehwfZf3FQd+3QZUw6/u7gtLRpG8OhnmRQH2IWnio8qd01fws+5ebx2Yx/ioiPcjmRqOSsgxpRDZFgIf7iyMyu272faT5vdjnOcF79Zw/drcpgwuAt9Wjd2O46pA6yAGFNOV3ZrzrltmvL816vZe6jA7TgAfJX5M6/+O4sbPAmMTGnldhxTR1gBMaacRIQJg7twIK+I575e7XYcsnYeZPxHS+kR35DHhnSxy9KaamMFxJgK6NgsmlF9WzPtp81kbM11LceBvEJufzeN8JAgJt3Uh4jQYNeymLrHCogxFXRP/w40iQzj0VR3Ljx19Khy3z+WsnH3YV4d2ZsWjepVewZTt/lVQERkgIisFpEsEXnwNP2GioiKiMd5HCoiU0VkuYisFJGHfPreIyKZIpIhIh+ISITT/l8RWeLctonIp077RSKS6/PcI2e26sacmYb1QnlgQCfSN+3ln4u3VvvrT/p+HV9l7uChKzpxbtum1f76xpRZQEQkGJgIXAEkAyNEJLmUftHAncACn+ZhQLiqdgP6ALeLSKKItHT6elS1KxAMDAdQ1fNVtaeq9gTmAZ/4LO+/Jc+p6uMVWF9jKtXQPvH0SGjEX75cxYG8wmp73Tmrd/Lc16sZ0rMFt56XVG2va4wvf7ZAUoAsVV2vqgXAdGBIKf2eAJ4FfM/zoECUiIQA9YACYL/zXAhQz3kuEtjmuzCnIF0CfOr/6hhTvYKChMcGdyHnQD6vfFc9F57avPswd01fQsezonn62u42aW5c408BaQls8Xmc7bQdIyK9gARVnXXC2BnAIWA7sBl4TlX3qOpW4DmnbTuQq6pfnzD2GuBbVd3v03auiCwVkS9FpEtpYUVktIikiUhaTk7NO/22qXl6JjTiek88b/2wgaydB6v0tY4UFDP63TRUlSmjPNQLs0lz4x5/Ckhpf94cmzEUkSDgRWB8Kf1SgGKgBZAEjBeRNiLSGO9WTJLzXJSI3HTC2BHABz6PFwGtVbUH8Aqn2DJR1Smq6lFVT2xsrB+rZ8yZu39AJ+qFBfPYzKqbUFdVHvxkGat3HODlEb1o1TSySl7HGH/5U0CygQSfx/Ecv7spGugKzBGRjUBfINWZSB8JzFbVQlXdCfwIeIDLgA2qmqOqhXjnOfqVLFBEmuItPp+XtKnqflU96Nz/AggVkZhyrq8xVSKmfjj3XNaB/67dxVeZO6rkNd76cSOfLdnG+P4duKhjXJW8hjHl4U8BWQi0F5EkEQnDO9mdWvKkquaqaoyqJqpqIjAfGKyqaXh3UV0iXlF4i8sqp72viESKdwfupYDvKU6HAbNU9dh8iog0c/oiIilO9t0VXnNjKtmoc1vT4az6/PnzFeQVFlfqsuet281TX6zk8uSz+N1F7Sp12cZUVJkFRFWLgHHAV3h/yX+kqpki8riIDC5j+ESgPpCBtxC9rarLVHUB3vmRRcByJ8cUn3HDOX73FcBQIENElgIvA8PVjS/fG3MKocFBTBjchey9R5j8/bpKW+62fUcYN20RiU0jef76HgTZZWlNgJDa/DvY4/FoWlqa2zFMHXPHtEX8a8UO/nXvhSQ0ObN5irzCYm54fR7rcg7x6R2/oF1c/UpKacypiUi6qnrK6mdHohtTyR4e2JkgEZ78/MwvPDUhNZOl2bk8N6yHFQ8TcKyAGFPJWjSqxx0Xt2V25s/8sHZXhZczbcFmpi/cwh0Xt2VA12aVmNCYymEFxJgq8Nvz29CqSSSPpmZQUHS03OMXbd7Lo6kZXNAhlnv7d6yChMacOSsgxlSBiNBgHhmUzLqcQ0ydu7FcY3ceyGPse+k0b1iPl4f3JNgmzU2AsgJiTBW5tHMcF3WM5f99u5ad+/PKHgAUFh9l3PuLyT1SyOSb+tAoMqyKUxpTcVZAjKkiIsKjV3WhoOgoT89e5deYJz9fyU8b9/DMdd1JbtGgihMac2asgBhThZJiorj1/CQ+WbSV9E17Ttv3n4uz+fvcjdzyiySG9Gx52r7GBAIrIMZUsXEXt6NZgwgeTc2k+Gjpx11lbM3lwY+Xc05SEx4a2KmaExpTMVZAjKliUeEhPDSwExlb9/Phwi0nPb/3UAFj3kunSVQYE2/sTWiw/bc0NYN9Uo2pBoN7tCAlqQl//WoV+w4XHGsvPqrcOX0xO/fnM+mmPsTUD3cxpTHlYwXEmGogIky4qgu5Rwp54Zs1x9qf+3o1/127iyeu7kLPhEYuJjSm/KyAGFNNkls04Ka+rXlv/iZWbNvPl8u3M2nOOkaktOKGs1u5Hc+YcgtxO4Axdcm9/Tswc+k27vvHUjbtPkSvVo2YMDjZ7VjGVIhtgRhTjRpFhnH/gE6s2L6femEhTLqxD+EhdllaUzPZFogx1ex6TwI/5+ZxWeezaNYwwu04xlSYFRBjqllwkHBP/w5uxzDmjNkuLGOMMRViBcQYY0yF+FVARGSAiKwWkSwRefA0/YaKiIqIx3kcKiJTRWS5iKwUkYd8+t4jIpkikiEiH4hIhNP+dxHZICJLnFtPp11E5GUnwzIR6X1mq26MMeZMlFlARCQYmAhcASQDI0TkpO8dikg0cCewwKd5GBCuqt2APsDtIpIoIi2dvh5V7QoEA8N9xv2fqvZ0bkuctiuA9s5tNDCpfKtqjDGmMvmzBZICZKnqelUtAKYDQ0rp9wTwLOB74QMFokQkBKgHFAD7nedCgHrOc5HAtjJyDAHeUa/5QCMRae5HfmOMMVXAnwLSEvA9A1y203aMiPQCElR11gljZwCHgO3AZuA5Vd2jqluB55y27UCuqn7tM+5JZzfViyJScnKgMnMYY4ypPv4UkNKup3nsnNQiEgS8CIwvpV8KUAy0AJKA8SLSRkQa492iSHKeixKRm5wxDwGdgLOBJsAD/uTwyTNaRNJEJC0nJ8eP1TPGGFMR/hSQbCDB53E8x+9uiga6AnNEZCPQF0h1JtJHArNVtVBVdwI/Ah7gMmCDquaoaiHwCdAPQFW3O7up8oG38RYhf3LgjJ+iqh5V9cTGxvqxesYYYyrCnwMJFwLtRSQJ2Ip3sntkyZOqmgvElDwWkTnAfaqaJiKXApeIyHt45zn6Ai/hnQ/pKyKRwBHgUiDNGd9cVbeLiABXAxnOolOBcSIyHTgH726v7acLnp6evktENvmxjqcSA+w6g/FVxXKVj+UqH8tVPrUxV2t/OpVZQFS1SETGAV/h/bbUW6qaKSKPA2mqmnqa4RPxbkVk4N0F9baqLgMQkRnAIqAIWAxMcca8LyKxTv8lwBin/QtgIJAFHAZ+40f2M9oEEZE0VfWcyTKqguUqH8tVPparfOpyLr9OZaKqX+D9Be7b9sgp+l7kc/8g3q/yltbvUeDRUtovOUV/Be7wJ68xxpiqZ0eiG2OMqRArIKc3pewurrBc5WO5ysdylU+dzSXePUPGGGNM+dgWiDHGmAqpkwWkrJNDisivRSTH54SOv/V57mYRWevcbg6gXMU+7af7Zlyl53L6XC8iK5wTZE7zaXft/Sojl2vvl3OGhZLXXiMi+3yec/Pzdbpcbr5frUTk3yKy2DlDxUCf5x5yxq0WkV8GQi7xnu/viM/7Nbmac7UWkW+dTHNEJN7nucr9fKlqnbrh/SryOqANEAYsBZJP6PNr4NVSxjYB1js/Gzv3G7udy3nuoIvvV3u8X8Vu7DyOC5D3q9Rcbr9fJ/T/Pd6vxrv+fp0ql9vvF959+WOd+8nARp/7S4FwvGe1WAcEB0CuRCDDxffrH8DNzv1LgHer6vNVF7dA/D05ZGl+CXyj3vN57QW+AQYEQK6q5E+u24CJznuCes86AO6/X6fKVZXK++84AvjAue/2+3WqXFXJn1wKNHDuN+R/Z6AYAkxX1XxV3YD3GLEUKseZ5KpK/uRKBr517v/b5/lK/3zVxQLi70kZr3M2AWeISMkpVKryhI5nkgsgQrznAJsvIldXUiZ/c3UAOojIj87rDyjHWDdygbvvF+Dd1YD3L+fvyju2mnOBu+/XBOAmEcnGezza78sx1o1cAEnOrq3vReT8Ssrkb66lwHXO/WuAaBFp6ufYcqmLBcSfkzLOBBJVtTvwL2BqOca6kQuglXqPOh0JvCQibasxVwje3UUX4f3L9Q0RaeTnWDdygbvvV4nhwAxVLa7A2PI6k1zg7vs1Avi7qsbjPRvFu+I9iavb79epcm3H+371Au4FpolIAyqHP7nuAy4UkcXAhXhPQVXk59hyqYsFpMyTMqrqbvWezBHgb3gvhuXXWJdyoarbnJ/rgTlAr+rK5fT5TL0nzdwArMb7i9vV9+s0udx+v0oM5/jdRG6/X6fK5fb7dSvwkfP684AIvOd5cvv9KjWXs0ttt9OejnfOokN15VLVbap6rVPAHnbacv1cp/KpiomeQL7h/at0Pd5N9JJJqC4n9Gnuc/8aYL7+bxJqA94JqMbO/SYBkKsx3is/gvc/1lpOM0FaBbkGAFN9Xn8L0DQA3q9T5XL1/XL6dQQ24hyLFQifr9Pkcvvz9SXwa+d+Z7y/9ATowvGT6OupvEn0M8kVW5ID72T31mr+3McAQc79J4HHq+rzdcYrVBNveDc31+D9y+Bhp+1xYLBz/y9ApvOP82+gk8/YW/BO1mUBvwmEXHhPhb/caV8O3FrNuQR4AVjhvP7wAHm/Ss3l9vvlPJ4APF3KWNfer1Plcvv9wjsp/KPz+kuAy33GPuyMWw1cEQi58M4/lPw/XQRcVc25huIt8muAN3CKf1V8vuxIdGOMMRVSF+dAjDHGVAIrIMYYYyrECogxxpgKsQJijDGmQqyAGGOMqRArIMYYYyrECogxxpgKsQJijDGmQv4/jaVY2jipyNYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "array([-0.49035519, -0.48973497, -0.48857292, -0.48943891, -0.4895049 ])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# examine the best model\n",
    "print(-grid_search.best_score_)\n",
    "print(grid_search.best_params_)\n",
    "# plot CV误差曲线\n",
    "test_means = grid_search.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid_search.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid_search.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid_search.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "x_axis = colsample_bytree\n",
    "\n",
    "plt.plot(x_axis, -test_means)\n",
    "#plt.errorbar(x_axis, -test_scores[:,i], yerr=test_stds[:,i] ,label = str(max_depths[i]) +' Test')\n",
    "#plt.errorbar(x_axis, -train_scores[:,i], yerr=train_stds[:,i] ,label = str(max_depths[i]) +' Train')\n",
    "\n",
    "plt.show()\n",
    "\n",
    "test_means"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 'colsample_bytree': 0.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best n_estimators: 1671\n",
      "best cv score: 0.47676236282797496\n"
     ]
    }
   ],
   "source": [
    "#较少学习率，调整基学习器\n",
    "params={'boosting_type':'gbdt',\n",
    "       'objective':'multiclass',\n",
    "       'n_jobs':-1,\n",
    "       'learning_rate':0.01,\n",
    "      # 'n_estimators':n_estimators_1,\n",
    "        'min_child_samples': 60,\n",
    "       'num_leaves':60,\n",
    "        'sub_samples': 0.5 ,\n",
    "        'maxdepth':7,\n",
    "       'max_bin':127,\n",
    "       'subsample':0.5,\n",
    "       'bagging_freq':1,\n",
    "       'colsample_bytree':0.7\n",
    "       }\n",
    "\n",
    "n_estimators_2 = get_n_estimators(params , X_train , y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(bagging_freq=1, boosting_type='gbdt', class_weight=None,\n",
       "        colsample_bytree=0.7, importance_type='split', learning_rate=0.01,\n",
       "        max_bin=127, max_depth=-1, maxdepth=7, min_child_samples=60,\n",
       "        min_child_weight=0.001, min_split_gain=0.0, n_estimators=100,\n",
       "        n_jobs=-1, num_leaves=60, objective='multiclass',\n",
       "        random_state=None, reg_alpha=0.0, reg_lambda=0.0, silent=False,\n",
       "        sub_samples=0.5, subsample=0.5, subsample_for_bin=200000,\n",
       "        subsample_freq=0)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#训练全体数据集\n",
    "params={'boosting_type':'gbdt',\n",
    "       'objective':'multiclass',\n",
    "       'n_jobs':-1,\n",
    "       'learning_rate':0.01,\n",
    "      # 'n_estimators':n_estimators_1,\n",
    "        'min_child_samples': 60,\n",
    "       'num_leaves':60,\n",
    "        'sub_samples': 0.5 ,\n",
    "        'maxdepth':7,\n",
    "       'max_bin':127,\n",
    "       'subsample':0.5,\n",
    "       'bagging_freq':1,\n",
    "       'colsample_bytree':0.7\n",
    "       }\n",
    "\n",
    "lg = LGBMClassifier(silent=False,  **params)\n",
    "lg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "pickle.dump(lg, open(\"Otto_LightGBM_gbdt_tfidf1.pkl\", 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({\"columns\":list(feat_names), \"importance\":list(lg.feature_importances_.T)})\n",
    "df = df.sort_values(by=['importance'],ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>columns</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>feat_67_tfidf</td>\n",
       "      <td>2460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>feat_25_tfidf</td>\n",
       "      <td>2100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>feat_40_tfidf</td>\n",
       "      <td>1991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>feat_24_tfidf</td>\n",
       "      <td>1951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>feat_86_tfidf</td>\n",
       "      <td>1813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>feat_48_tfidf</td>\n",
       "      <td>1734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>feat_14_tfidf</td>\n",
       "      <td>1600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>feat_42_tfidf</td>\n",
       "      <td>1539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>feat_60_tfidf</td>\n",
       "      <td>1457</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>feat_15_tfidf</td>\n",
       "      <td>1385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>feat_62_tfidf</td>\n",
       "      <td>1280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>feat_34_tfidf</td>\n",
       "      <td>1231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>feat_88_tfidf</td>\n",
       "      <td>1206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>feat_43_tfidf</td>\n",
       "      <td>1131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>feat_32_tfidf</td>\n",
       "      <td>1024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>feat_64_tfidf</td>\n",
       "      <td>971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>feat_8_tfidf</td>\n",
       "      <td>905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>feat_16_tfidf</td>\n",
       "      <td>888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>feat_36_tfidf</td>\n",
       "      <td>866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>feat_26_tfidf</td>\n",
       "      <td>863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>feat_33_tfidf</td>\n",
       "      <td>849</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>feat_71_tfidf</td>\n",
       "      <td>848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>feat_92_tfidf</td>\n",
       "      <td>829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>feat_11_tfidf</td>\n",
       "      <td>811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>feat_9_tfidf</td>\n",
       "      <td>752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>feat_72_tfidf</td>\n",
       "      <td>713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>feat_70_tfidf</td>\n",
       "      <td>707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>feat_85_tfidf</td>\n",
       "      <td>680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>feat_17_tfidf</td>\n",
       "      <td>666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>feat_75_tfidf</td>\n",
       "      <td>651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>feat_4_tfidf</td>\n",
       "      <td>221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>feat_89_tfidf</td>\n",
       "      <td>220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>feat_10_tfidf</td>\n",
       "      <td>217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>feat_80_tfidf</td>\n",
       "      <td>203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>feat_74_tfidf</td>\n",
       "      <td>198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>feat_73_tfidf</td>\n",
       "      <td>198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>feat_27_tfidf</td>\n",
       "      <td>195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>feat_45_tfidf</td>\n",
       "      <td>190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>feat_77_tfidf</td>\n",
       "      <td>187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>feat_23_tfidf</td>\n",
       "      <td>171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>feat_81_tfidf</td>\n",
       "      <td>153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>feat_19_tfidf</td>\n",
       "      <td>150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>feat_3_tfidf</td>\n",
       "      <td>141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>feat_84_tfidf</td>\n",
       "      <td>136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>feat_49_tfidf</td>\n",
       "      <td>107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>feat_46_tfidf</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>feat_2_tfidf</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>feat_93_tfidf</td>\n",
       "      <td>99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>feat_65_tfidf</td>\n",
       "      <td>94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>feat_61_tfidf</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>feat_21_tfidf</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>feat_5_tfidf</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>feat_31_tfidf</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>feat_63_tfidf</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>feat_12_tfidf</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>feat_6_tfidf</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>feat_51_tfidf</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>feat_52_tfidf</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>feat_28_tfidf</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>feat_82_tfidf</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>93 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          columns  importance\n",
       "66  feat_67_tfidf        2460\n",
       "24  feat_25_tfidf        2100\n",
       "39  feat_40_tfidf        1991\n",
       "23  feat_24_tfidf        1951\n",
       "85  feat_86_tfidf        1813\n",
       "47  feat_48_tfidf        1734\n",
       "13  feat_14_tfidf        1600\n",
       "41  feat_42_tfidf        1539\n",
       "59  feat_60_tfidf        1457\n",
       "14  feat_15_tfidf        1385\n",
       "61  feat_62_tfidf        1280\n",
       "33  feat_34_tfidf        1231\n",
       "87  feat_88_tfidf        1206\n",
       "42  feat_43_tfidf        1131\n",
       "31  feat_32_tfidf        1024\n",
       "63  feat_64_tfidf         971\n",
       "7    feat_8_tfidf         905\n",
       "15  feat_16_tfidf         888\n",
       "35  feat_36_tfidf         866\n",
       "25  feat_26_tfidf         863\n",
       "32  feat_33_tfidf         849\n",
       "70  feat_71_tfidf         848\n",
       "91  feat_92_tfidf         829\n",
       "10  feat_11_tfidf         811\n",
       "8    feat_9_tfidf         752\n",
       "71  feat_72_tfidf         713\n",
       "69  feat_70_tfidf         707\n",
       "84  feat_85_tfidf         680\n",
       "16  feat_17_tfidf         666\n",
       "74  feat_75_tfidf         651\n",
       "..            ...         ...\n",
       "3    feat_4_tfidf         221\n",
       "88  feat_89_tfidf         220\n",
       "9   feat_10_tfidf         217\n",
       "79  feat_80_tfidf         203\n",
       "73  feat_74_tfidf         198\n",
       "72  feat_73_tfidf         198\n",
       "26  feat_27_tfidf         195\n",
       "44  feat_45_tfidf         190\n",
       "76  feat_77_tfidf         187\n",
       "22  feat_23_tfidf         171\n",
       "80  feat_81_tfidf         153\n",
       "18  feat_19_tfidf         150\n",
       "2    feat_3_tfidf         141\n",
       "83  feat_84_tfidf         136\n",
       "48  feat_49_tfidf         107\n",
       "45  feat_46_tfidf         100\n",
       "1    feat_2_tfidf         100\n",
       "92  feat_93_tfidf          99\n",
       "64  feat_65_tfidf          94\n",
       "60  feat_61_tfidf          80\n",
       "20  feat_21_tfidf          57\n",
       "4    feat_5_tfidf          49\n",
       "30  feat_31_tfidf          48\n",
       "62  feat_63_tfidf          42\n",
       "11  feat_12_tfidf          42\n",
       "5    feat_6_tfidf          38\n",
       "50  feat_51_tfidf          35\n",
       "51  feat_52_tfidf          29\n",
       "27  feat_28_tfidf          25\n",
       "81  feat_82_tfidf           5\n",
       "\n",
       "[93 rows x 2 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>feat_1_tfidf</th>\n",
       "      <th>feat_2_tfidf</th>\n",
       "      <th>feat_3_tfidf</th>\n",
       "      <th>feat_4_tfidf</th>\n",
       "      <th>feat_5_tfidf</th>\n",
       "      <th>feat_6_tfidf</th>\n",
       "      <th>feat_7_tfidf</th>\n",
       "      <th>feat_8_tfidf</th>\n",
       "      <th>feat_9_tfidf</th>\n",
       "      <th>...</th>\n",
       "      <th>feat_84_tfidf</th>\n",
       "      <th>feat_85_tfidf</th>\n",
       "      <th>feat_86_tfidf</th>\n",
       "      <th>feat_87_tfidf</th>\n",
       "      <th>feat_88_tfidf</th>\n",
       "      <th>feat_89_tfidf</th>\n",
       "      <th>feat_90_tfidf</th>\n",
       "      <th>feat_91_tfidf</th>\n",
       "      <th>feat_92_tfidf</th>\n",
       "      <th>feat_93_tfidf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.421803</td>\n",
       "      <td>0.052224</td>\n",
       "      <td>0.842245</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.068759</td>\n",
       "      <td>0.094897</td>\n",
       "      <td>0.502059</td>\n",
       "      <td>0.535424</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.143963</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.070171</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.071486</td>\n",
       "      <td>0.648348</td>\n",
       "      <td>0.050417</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.078248</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.071995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.048481</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.139311</td>\n",
       "      <td>0.034257</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0.048446</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.047156</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.065018</td>\n",
       "      <td>0.081172</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.556178</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 94 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  feat_1_tfidf  feat_2_tfidf  feat_3_tfidf  feat_4_tfidf  feat_5_tfidf  \\\n",
       "0   1      0.000000      0.000000      0.000000      0.000000           0.0   \n",
       "1   2      0.068759      0.094897      0.502059      0.535424           0.0   \n",
       "2   3      0.000000      0.071486      0.648348      0.050417           0.0   \n",
       "3   4      0.000000      0.000000      0.000000      0.048481           0.0   \n",
       "4   5      0.048446      0.000000      0.000000      0.047156           0.0   \n",
       "\n",
       "   feat_6_tfidf  feat_7_tfidf  feat_8_tfidf  feat_9_tfidf      ...        \\\n",
       "0           0.0      0.000000      0.000000           0.0      ...         \n",
       "1           0.0      0.000000      0.000000           0.0      ...         \n",
       "2           0.0      0.000000      0.000000           0.0      ...         \n",
       "3           0.0      0.000000      0.000000           0.0      ...         \n",
       "4           0.0      0.065018      0.081172           0.0      ...         \n",
       "\n",
       "   feat_84_tfidf  feat_85_tfidf  feat_86_tfidf  feat_87_tfidf  feat_88_tfidf  \\\n",
       "0            0.0       0.000000       0.421803       0.052224       0.842245   \n",
       "1            0.0       0.000000       0.000000       0.000000       0.000000   \n",
       "2            0.0       0.000000       0.000000       0.000000       0.078248   \n",
       "3            0.0       0.139311       0.034257       0.000000       0.000000   \n",
       "4            0.0       0.000000       0.000000       0.000000       0.000000   \n",
       "\n",
       "   feat_89_tfidf  feat_90_tfidf  feat_91_tfidf  feat_92_tfidf  feat_93_tfidf  \n",
       "0       0.000000            0.0       0.000000       0.000000       0.000000  \n",
       "1       0.143963            0.0       0.000000       0.070171       0.000000  \n",
       "2       0.000000            0.0       0.000000       0.000000       0.071995  \n",
       "3       0.000000            0.0       0.000000       0.000000       0.000000  \n",
       "4       0.000000            0.0       0.556178       0.000000       0.000000  \n",
       "\n",
       "[5 rows x 94 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test=pd.read_csv(\"Otto_FE_test_tfidf.csv\")\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(144368, 9)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_id=test['id']\n",
    "X_test=test.drop([\"id\"],axis=1)\n",
    "#保留特征名字\n",
    "feat_names=X_test.columns\n",
    "#数据用稀疏矩阵表示，模型训练变快\n",
    "from scipy.sparse import csr_matrix\n",
    "X_test=csr_matrix(X_test)\n",
    "import pickle\n",
    "LightGBM_gbdt=pickle.load(open(\"Otto_LightGBM_gbdt_tfidf1.pkl\",'rb'))\n",
    "y_test_pred=LightGBM_gbdt.predict_proba(X_test)\n",
    "y_test_pred.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#生成提交结果\n",
    "out_df=pd.DataFrame(y_test_pred)\n",
    "columns=np.empty(9,dtype=object)\n",
    "for i in range(9):\n",
    "    columns[i]='Class_'+str(i+1)\n",
    "out_df.columns=columns\n",
    "out_df=pd.concat([test_id,out_df],axis=1)\n",
    "out_df.to_csv(\"Otto_LightGBM_gbdt_tfidf1.csv\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "![image2]\n",
    "\n",
    "![image1]\n",
    "\n",
    "[image2]:\n",
    "\n",
    "[image1]:\n",
    "\n",
    "[image1]:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "比赛结束，没有自己排名，查看排行榜，得分排名在746名的样子"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
